diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,4833 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.04014452027298274, + "eval_steps": 500, + "global_step": 300, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "completion_length": 564.25, + "epoch": 0.00013381506757660912, + "grad_norm": 0.14033202826976776, + "kl": 0.0, + "learning_rate": 8.88888888888889e-08, + "loss": 0.0, + "reward": -0.6452499628067017, + "reward_std": 0.8964393734931946, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.7702499628067017, + "step": 1 + }, + { + "completion_length": 523.0, + "epoch": 0.00026763013515321824, + "grad_norm": 0.1592259407043457, + "kl": 0.0, + "learning_rate": 1.777777777777778e-07, + "loss": -0.0, + "reward": -0.9787499904632568, + "reward_std": 1.8240368366241455, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.9787499904632568, + "step": 2 + }, + { + "completion_length": 264.25, + "epoch": 0.0004014452027298274, + "grad_norm": 0.1932271420955658, + "kl": 6.622826731472742e-06, + "learning_rate": 2.666666666666667e-07, + "loss": 0.0, + "reward": -0.226500004529953, + "reward_std": 0.26771190762519836, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.226500004529953, + "step": 3 + }, + { + "completion_length": 159.0, + "epoch": 0.0005352602703064365, + "grad_norm": 0.27002546191215515, + "kl": 1.078558216249803e-05, + "learning_rate": 3.555555555555556e-07, + "loss": 0.0, + "reward": 0.013750001788139343, + "reward_std": 0.2580934762954712, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.11124999821186066, + "step": 4 + }, + { + "completion_length": 320.75, + "epoch": 0.0006690753378830456, + "grad_norm": 0.1831292062997818, + "kl": 8.28549855214078e-06, + "learning_rate": 4.444444444444445e-07, + "loss": 0.0, + "reward": 0.0624999925494194, + "reward_std": 0.21419848501682281, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.0625, + "step": 5 + }, + { + "completion_length": 237.25, + "epoch": 0.0008028904054596548, + "grad_norm": 0.18136273324489594, + "kl": 5.389752914197743e-06, + "learning_rate": 5.333333333333335e-07, + "loss": 0.0, + "reward": -0.08349999785423279, + "reward_std": 0.16699999570846558, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.08349999785423279, + "step": 6 + }, + { + "completion_length": 375.25, + "epoch": 0.0009367054730362638, + "grad_norm": 0.18059618771076202, + "kl": 1.1201269444427453e-05, + "learning_rate": 6.222222222222223e-07, + "loss": 0.0, + "reward": -0.45899999141693115, + "reward_std": 0.9180000424385071, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.45899999141693115, + "step": 7 + }, + { + "completion_length": 277.0, + "epoch": 0.001070520540612873, + "grad_norm": 0.19177855551242828, + "kl": 5.332793080015108e-06, + "learning_rate": 7.111111111111112e-07, + "loss": 0.0, + "reward": -0.0560000017285347, + "reward_std": 0.1120000034570694, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.0560000017285347, + "step": 8 + }, + { + "completion_length": 334.5, + "epoch": 0.0012043356081894822, + "grad_norm": 0.27444255352020264, + "kl": 1.158629765996011e-05, + "learning_rate": 8.000000000000001e-07, + "loss": 0.0, + "reward": 0.2122499942779541, + "reward_std": 0.39513909816741943, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0872499942779541, + "step": 9 + }, + { + "completion_length": 297.25, + "epoch": 0.0013381506757660913, + "grad_norm": 0.20776011049747467, + "kl": 8.035244718485046e-06, + "learning_rate": 8.88888888888889e-07, + "loss": 0.0, + "reward": 0.5809999704360962, + "reward_std": 1.0095866918563843, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.6690000295639038, + "step": 10 + }, + { + "completion_length": 236.5, + "epoch": 0.0014719657433427003, + "grad_norm": 9.880962898023427e-05, + "kl": 7.1510494308313355e-06, + "learning_rate": 9.77777777777778e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 11 + }, + { + "completion_length": 342.25, + "epoch": 0.0016057808109193096, + "grad_norm": 0.11707701534032822, + "kl": 5.569832410401432e-06, + "learning_rate": 1.066666666666667e-06, + "loss": 0.0, + "reward": -0.4452499747276306, + "reward_std": 0.3653622269630432, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.5702500343322754, + "step": 12 + }, + { + "completion_length": 261.0, + "epoch": 0.0017395958784959186, + "grad_norm": 0.22486890852451324, + "kl": 7.698228728258982e-06, + "learning_rate": 1.1555555555555556e-06, + "loss": 0.0, + "reward": -0.09549999237060547, + "reward_std": 0.451213538646698, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.22049999237060547, + "step": 13 + }, + { + "completion_length": 289.5, + "epoch": 0.0018734109460725277, + "grad_norm": 0.0001465526584070176, + "kl": 8.079272447503172e-06, + "learning_rate": 1.2444444444444445e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 14 + }, + { + "completion_length": 467.75, + "epoch": 0.002007226013649137, + "grad_norm": 7.054989691823721e-05, + "kl": 8.345767128048465e-06, + "learning_rate": 1.3333333333333334e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 15 + }, + { + "completion_length": 242.5, + "epoch": 0.002141041081225746, + "grad_norm": 0.20067830383777618, + "kl": 4.895833171758568e-06, + "learning_rate": 1.4222222222222223e-06, + "loss": 0.0, + "reward": 0.5694999694824219, + "reward_std": 0.9794492721557617, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.05550000071525574, + "step": 16 + }, + { + "completion_length": 397.75, + "epoch": 0.002274856148802355, + "grad_norm": 6.06259964115452e-05, + "kl": 7.802555046509951e-06, + "learning_rate": 1.5111111111111112e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 17 + }, + { + "completion_length": 263.75, + "epoch": 0.0024086712163789645, + "grad_norm": 0.2957823872566223, + "kl": 8.453114787698723e-06, + "learning_rate": 1.6000000000000001e-06, + "loss": 0.0, + "reward": -0.012249999679625034, + "reward_std": 0.02449999935925007, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.012249999679625034, + "step": 18 + }, + { + "completion_length": 301.0, + "epoch": 0.0025424862839555735, + "grad_norm": 0.277434378862381, + "kl": 8.06238858785946e-06, + "learning_rate": 1.688888888888889e-06, + "loss": 0.0, + "reward": 0.1875, + "reward_std": 0.375, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0625, + "step": 19 + }, + { + "completion_length": 255.75, + "epoch": 0.0026763013515321826, + "grad_norm": 9.750338358571753e-05, + "kl": 7.102700237737736e-06, + "learning_rate": 1.777777777777778e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 20 + }, + { + "completion_length": 200.5, + "epoch": 0.0028101164191087916, + "grad_norm": 0.29159611463546753, + "kl": 8.880564564606175e-06, + "learning_rate": 1.8666666666666669e-06, + "loss": 0.0, + "reward": -0.1912499964237213, + "reward_std": 0.13046424090862274, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.1912499964237213, + "step": 21 + }, + { + "completion_length": 265.0, + "epoch": 0.0029439314866854006, + "grad_norm": 0.31326013803482056, + "kl": 1.243360384250991e-05, + "learning_rate": 1.955555555555556e-06, + "loss": 0.0, + "reward": -0.25849997997283936, + "reward_std": 0.8479105830192566, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.38349997997283936, + "step": 22 + }, + { + "completion_length": 154.0, + "epoch": 0.00307774655426201, + "grad_norm": 0.2559646666049957, + "kl": 5.530042471946217e-06, + "learning_rate": 2.0444444444444447e-06, + "loss": 0.0, + "reward": -0.05624999850988388, + "reward_std": 0.11249999701976776, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.05624999850988388, + "step": 23 + }, + { + "completion_length": 189.75, + "epoch": 0.003211561621838619, + "grad_norm": 0.284862220287323, + "kl": 9.174956176138949e-06, + "learning_rate": 2.133333333333334e-06, + "loss": 0.0, + "reward": 0.1277499943971634, + "reward_std": 0.2554999887943268, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.002749999985098839, + "step": 24 + }, + { + "completion_length": 268.75, + "epoch": 0.003345376689415228, + "grad_norm": 0.15809978544712067, + "kl": 4.42105692854966e-06, + "learning_rate": 2.222222222222222e-06, + "loss": 0.0, + "reward": -0.2212499976158142, + "reward_std": 0.4424999952316284, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.2212499976158142, + "step": 25 + }, + { + "completion_length": 210.0, + "epoch": 0.0034791917569918372, + "grad_norm": 0.2747434675693512, + "kl": 8.758857802604325e-06, + "learning_rate": 2.311111111111111e-06, + "loss": 0.0, + "reward": 0.6607499718666077, + "reward_std": 1.2395679950714111, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.035750001668930054, + "step": 26 + }, + { + "completion_length": 437.25, + "epoch": 0.0036130068245684463, + "grad_norm": 0.18731197714805603, + "kl": 1.347947727481369e-05, + "learning_rate": 2.4000000000000003e-06, + "loss": 0.0, + "reward": -0.06724999845027924, + "reward_std": 0.13449999690055847, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.06724999845027924, + "step": 27 + }, + { + "completion_length": 307.75, + "epoch": 0.0037468218921450553, + "grad_norm": 0.21840307116508484, + "kl": 8.306662493851036e-06, + "learning_rate": 2.488888888888889e-06, + "loss": 0.0, + "reward": -0.07774999737739563, + "reward_std": 0.24599508941173553, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.07774999737739563, + "step": 28 + }, + { + "completion_length": 620.25, + "epoch": 0.003880636959721665, + "grad_norm": 4.864737275056541e-05, + "kl": 6.508589649456553e-06, + "learning_rate": 2.577777777777778e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 29 + }, + { + "completion_length": 292.75, + "epoch": 0.004014452027298274, + "grad_norm": 0.4136326313018799, + "kl": 1.4081160770729184e-05, + "learning_rate": 2.666666666666667e-06, + "loss": 0.0, + "reward": -0.05950000137090683, + "reward_std": 0.30902159214019775, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.05950000137090683, + "step": 30 + }, + { + "completion_length": 398.25, + "epoch": 0.004148267094874883, + "grad_norm": 4.4143911509308964e-05, + "kl": 5.329064151737839e-06, + "learning_rate": 2.755555555555556e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 31 + }, + { + "completion_length": 262.5, + "epoch": 0.004282082162451492, + "grad_norm": 5.927432721364312e-05, + "kl": 4.8590136429993436e-06, + "learning_rate": 2.8444444444444446e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 32 + }, + { + "completion_length": 307.0, + "epoch": 0.004415897230028101, + "grad_norm": 0.19121825695037842, + "kl": 6.706204203510424e-06, + "learning_rate": 2.9333333333333338e-06, + "loss": 0.0, + "reward": -0.3400000333786011, + "reward_std": 0.5228951573371887, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.4650000333786011, + "step": 33 + }, + { + "completion_length": 463.5, + "epoch": 0.00454971229760471, + "grad_norm": 5.6564931583125144e-05, + "kl": 4.871402779826894e-06, + "learning_rate": 3.0222222222222225e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 34 + }, + { + "completion_length": 352.25, + "epoch": 0.004683527365181319, + "grad_norm": 0.1656392216682434, + "kl": 6.969175501581049e-06, + "learning_rate": 3.1111111111111116e-06, + "loss": 0.0, + "reward": 0.37549999356269836, + "reward_std": 0.7509999871253967, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.24950000643730164, + "step": 35 + }, + { + "completion_length": 388.75, + "epoch": 0.004817342432757929, + "grad_norm": 5.8849836932495236e-05, + "kl": 7.587910658912733e-06, + "learning_rate": 3.2000000000000003e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 36 + }, + { + "completion_length": 272.75, + "epoch": 0.004951157500334538, + "grad_norm": 0.19752717018127441, + "kl": 9.09756181499688e-06, + "learning_rate": 3.2888888888888894e-06, + "loss": 0.0, + "reward": 0.06274999678134918, + "reward_std": 0.12549999356269836, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.06274999678134918, + "step": 37 + }, + { + "completion_length": 265.25, + "epoch": 0.005084972567911147, + "grad_norm": 0.1716996431350708, + "kl": 7.701055437792093e-06, + "learning_rate": 3.377777777777778e-06, + "loss": 0.0, + "reward": -0.17024999856948853, + "reward_std": 0.197299063205719, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.2952499985694885, + "step": 38 + }, + { + "completion_length": 198.25, + "epoch": 0.005218787635487756, + "grad_norm": 8.153666567523032e-05, + "kl": 7.68154040997615e-06, + "learning_rate": 3.4666666666666672e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 39 + }, + { + "completion_length": 357.75, + "epoch": 0.005352602703064365, + "grad_norm": 0.16188818216323853, + "kl": 8.425393389188685e-06, + "learning_rate": 3.555555555555556e-06, + "loss": 0.0, + "reward": -0.09000000357627869, + "reward_std": 0.18000000715255737, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.09000000357627869, + "step": 40 + }, + { + "completion_length": 322.5, + "epoch": 0.005486417770640974, + "grad_norm": 0.22530034184455872, + "kl": 1.0753658898465801e-05, + "learning_rate": 3.644444444444445e-06, + "loss": 0.0, + "reward": 0.045249998569488525, + "reward_std": 0.09049999713897705, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.07975000143051147, + "step": 41 + }, + { + "completion_length": 247.75, + "epoch": 0.005620232838217583, + "grad_norm": 0.00012052639067405835, + "kl": 8.426506610703655e-06, + "learning_rate": 3.7333333333333337e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 42 + }, + { + "completion_length": 354.25, + "epoch": 0.005754047905794192, + "grad_norm": 0.21594372391700745, + "kl": 1.0382359505456407e-05, + "learning_rate": 3.8222222222222224e-06, + "loss": 0.0, + "reward": 0.11999999731779099, + "reward_std": 0.1444529891014099, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.11999999731779099, + "step": 43 + }, + { + "completion_length": 212.25, + "epoch": 0.005887862973370801, + "grad_norm": 0.00011440912930993363, + "kl": 7.224463388411095e-06, + "learning_rate": 3.911111111111112e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 44 + }, + { + "completion_length": 249.75, + "epoch": 0.00602167804094741, + "grad_norm": 0.20934244990348816, + "kl": 1.420614898961503e-05, + "learning_rate": 4.000000000000001e-06, + "loss": 0.0, + "reward": 0.06699999421834946, + "reward_std": 0.12292815744876862, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.06699999421834946, + "step": 45 + }, + { + "completion_length": 273.0, + "epoch": 0.00615549310852402, + "grad_norm": 0.2055320143699646, + "kl": 1.246560350409709e-05, + "learning_rate": 4.088888888888889e-06, + "loss": 0.0, + "reward": 0.43024998903274536, + "reward_std": 0.8604999780654907, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.19474999606609344, + "step": 46 + }, + { + "completion_length": 350.25, + "epoch": 0.006289308176100629, + "grad_norm": 0.18098776042461395, + "kl": 1.056162545864936e-05, + "learning_rate": 4.177777777777778e-06, + "loss": 0.0, + "reward": -0.20074999332427979, + "reward_std": 0.40149998664855957, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.3257499933242798, + "step": 47 + }, + { + "completion_length": 282.25, + "epoch": 0.006423123243677238, + "grad_norm": 0.2726364731788635, + "kl": 1.3284003216540441e-05, + "learning_rate": 4.266666666666668e-06, + "loss": 0.0, + "reward": -0.6389999985694885, + "reward_std": 0.7382077574729919, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.6389999985694885, + "step": 48 + }, + { + "completion_length": 165.5, + "epoch": 0.006556938311253847, + "grad_norm": 0.439759224653244, + "kl": 1.5033414456411265e-05, + "learning_rate": 4.3555555555555555e-06, + "loss": 0.0, + "reward": 0.5855000019073486, + "reward_std": 1.1710000038146973, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.039500001817941666, + "step": 49 + }, + { + "completion_length": 401.0, + "epoch": 0.006690753378830456, + "grad_norm": 0.1494322270154953, + "kl": 1.3026328815612942e-05, + "learning_rate": 4.444444444444444e-06, + "loss": 0.0, + "reward": -0.11124999821186066, + "reward_std": 0.2224999964237213, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.11124999821186066, + "step": 50 + }, + { + "completion_length": 205.25, + "epoch": 0.006824568446407065, + "grad_norm": 0.23992237448692322, + "kl": 1.3739524547418114e-05, + "learning_rate": 4.533333333333334e-06, + "loss": 0.0, + "reward": 0.03125, + "reward_std": 0.0625, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.03125, + "step": 51 + }, + { + "completion_length": 187.25, + "epoch": 0.0069583835139836745, + "grad_norm": 0.2555287480354309, + "kl": 1.5424680896103382e-05, + "learning_rate": 4.622222222222222e-06, + "loss": 0.0, + "reward": -0.07625000178813934, + "reward_std": 0.12490096688270569, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.07625000178813934, + "step": 52 + }, + { + "completion_length": 230.0, + "epoch": 0.0070921985815602835, + "grad_norm": 0.00012860310380347073, + "kl": 1.0372207725595217e-05, + "learning_rate": 4.711111111111111e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 53 + }, + { + "completion_length": 299.25, + "epoch": 0.0072260136491368926, + "grad_norm": 0.00015418548719026148, + "kl": 1.1093783541582525e-05, + "learning_rate": 4.800000000000001e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 54 + }, + { + "completion_length": 392.25, + "epoch": 0.007359828716713502, + "grad_norm": 0.0001652293576626107, + "kl": 1.3498687621904537e-05, + "learning_rate": 4.888888888888889e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 55 + }, + { + "completion_length": 316.5, + "epoch": 0.007493643784290111, + "grad_norm": 0.14983882009983063, + "kl": 2.462963675498031e-05, + "learning_rate": 4.977777777777778e-06, + "loss": 0.0, + "reward": -0.3332500159740448, + "reward_std": 0.4945647418498993, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.3332500159740448, + "step": 56 + }, + { + "completion_length": 219.0, + "epoch": 0.0076274588518667205, + "grad_norm": 0.2960438132286072, + "kl": 3.720477252500132e-05, + "learning_rate": 5.0666666666666676e-06, + "loss": 0.0, + "reward": -0.13199999928474426, + "reward_std": 0.2639999985694885, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.13199999928474426, + "step": 57 + }, + { + "completion_length": 235.5, + "epoch": 0.00776127391944333, + "grad_norm": 0.000978139229118824, + "kl": 3.992651545559056e-05, + "learning_rate": 5.155555555555556e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 58 + }, + { + "completion_length": 249.0, + "epoch": 0.007895088987019938, + "grad_norm": 0.00023408984998241067, + "kl": 2.2337197151500732e-05, + "learning_rate": 5.244444444444445e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 59 + }, + { + "completion_length": 263.25, + "epoch": 0.008028904054596548, + "grad_norm": 0.2353731095790863, + "kl": 3.2109041057992727e-05, + "learning_rate": 5.333333333333334e-06, + "loss": 0.0, + "reward": -0.030500000342726707, + "reward_std": 0.061000000685453415, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.030500000342726707, + "step": 60 + }, + { + "completion_length": 527.75, + "epoch": 0.008162719122173156, + "grad_norm": 0.15068306028842926, + "kl": 2.8251575713511556e-05, + "learning_rate": 5.422222222222223e-06, + "loss": 0.0, + "reward": -0.27649998664855957, + "reward_std": 0.5227195024490356, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.27649998664855957, + "step": 61 + }, + { + "completion_length": 243.75, + "epoch": 0.008296534189749766, + "grad_norm": 0.21375156939029694, + "kl": 5.114181476528756e-05, + "learning_rate": 5.511111111111112e-06, + "loss": 0.0, + "reward": -0.06224999949336052, + "reward_std": 0.12449999898672104, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.06224999949336052, + "step": 62 + }, + { + "completion_length": 197.5, + "epoch": 0.008430349257326376, + "grad_norm": 0.20765534043312073, + "kl": 6.65866318740882e-05, + "learning_rate": 5.600000000000001e-06, + "loss": 0.0, + "reward": 0.06274999678134918, + "reward_std": 0.12549999356269836, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.06274999678134918, + "step": 63 + }, + { + "completion_length": 415.75, + "epoch": 0.008564164324902984, + "grad_norm": 0.00047714909305796027, + "kl": 5.4172531235963106e-05, + "learning_rate": 5.688888888888889e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 64 + }, + { + "completion_length": 561.25, + "epoch": 0.008697979392479594, + "grad_norm": 0.00017319328617304564, + "kl": 2.3787781174178235e-05, + "learning_rate": 5.777777777777778e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 65 + }, + { + "completion_length": 266.0, + "epoch": 0.008831794460056202, + "grad_norm": 0.18374453485012054, + "kl": 7.648408063687384e-05, + "learning_rate": 5.8666666666666675e-06, + "loss": 0.0, + "reward": -0.2854999899864197, + "reward_std": 0.40305209159851074, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.4104999899864197, + "step": 66 + }, + { + "completion_length": 367.0, + "epoch": 0.008965609527632812, + "grad_norm": 0.00041845859959721565, + "kl": 6.105724605731666e-05, + "learning_rate": 5.955555555555555e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 67 + }, + { + "completion_length": 165.25, + "epoch": 0.00909942459520942, + "grad_norm": 0.3174028694629669, + "kl": 0.00012760543904732913, + "learning_rate": 6.044444444444445e-06, + "loss": 0.0, + "reward": -0.09300000220537186, + "reward_std": 0.10758562386035919, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.09300000220537186, + "step": 68 + }, + { + "completion_length": 261.75, + "epoch": 0.00923323966278603, + "grad_norm": 0.0009887454798445106, + "kl": 0.00011684713535942137, + "learning_rate": 6.133333333333334e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 69 + }, + { + "completion_length": 351.25, + "epoch": 0.009367054730362638, + "grad_norm": 0.22120727598667145, + "kl": 8.070362673606724e-05, + "learning_rate": 6.222222222222223e-06, + "loss": 0.0, + "reward": 0.09224999696016312, + "reward_std": 0.18449999392032623, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.09224999696016312, + "step": 70 + }, + { + "completion_length": 364.75, + "epoch": 0.009500869797939248, + "grad_norm": 0.14219215512275696, + "kl": 0.00014313386054709554, + "learning_rate": 6.311111111111111e-06, + "loss": 0.0, + "reward": 0.03125, + "reward_std": 0.0625, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.03125, + "step": 71 + }, + { + "completion_length": 301.0, + "epoch": 0.009634684865515858, + "grad_norm": 0.00125442526768893, + "kl": 0.00022049600374884903, + "learning_rate": 6.4000000000000006e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 72 + }, + { + "completion_length": 257.5, + "epoch": 0.009768499933092466, + "grad_norm": 0.27416515350341797, + "kl": 0.0003145383088849485, + "learning_rate": 6.488888888888889e-06, + "loss": 0.0, + "reward": 0.20900002121925354, + "reward_std": 0.4749357998371124, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.16599999368190765, + "step": 73 + }, + { + "completion_length": 355.75, + "epoch": 0.009902315000669076, + "grad_norm": 0.0006047863862477243, + "kl": 9.019898425322026e-05, + "learning_rate": 6.577777777777779e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 74 + }, + { + "completion_length": 329.25, + "epoch": 0.010036130068245684, + "grad_norm": 0.19630220532417297, + "kl": 0.00013932358706369996, + "learning_rate": 6.666666666666667e-06, + "loss": 0.0, + "reward": -0.3462499976158142, + "reward_std": 0.6924999952316284, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.3462499976158142, + "step": 75 + }, + { + "completion_length": 203.0, + "epoch": 0.010169945135822294, + "grad_norm": 0.19907738268375397, + "kl": 0.0003505215863697231, + "learning_rate": 6.755555555555556e-06, + "loss": 0.0, + "reward": -0.08375000208616257, + "reward_std": 0.16750000417232513, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.08375000208616257, + "step": 76 + }, + { + "completion_length": 156.0, + "epoch": 0.010303760203398902, + "grad_norm": 0.0033248290419578552, + "kl": 0.0005173031822778285, + "learning_rate": 6.844444444444445e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 77 + }, + { + "completion_length": 248.5, + "epoch": 0.010437575270975512, + "grad_norm": 0.2999407947063446, + "kl": 0.0002213937696069479, + "learning_rate": 6.9333333333333344e-06, + "loss": 0.0, + "reward": 0.125, + "reward_std": 0.25, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 78 + }, + { + "completion_length": 243.0, + "epoch": 0.01057139033855212, + "grad_norm": 0.2764039933681488, + "kl": 0.00041108846198767424, + "learning_rate": 7.022222222222222e-06, + "loss": 0.0, + "reward": -0.08399999886751175, + "reward_std": 0.1679999977350235, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.08399999886751175, + "step": 79 + }, + { + "completion_length": 577.0, + "epoch": 0.01070520540612873, + "grad_norm": 0.1485264003276825, + "kl": 0.0002135551767423749, + "learning_rate": 7.111111111111112e-06, + "loss": 0.0, + "reward": -0.6702499985694885, + "reward_std": 1.340499997138977, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.6702499985694885, + "step": 80 + }, + { + "completion_length": 491.75, + "epoch": 0.010839020473705338, + "grad_norm": 0.0006590148550458252, + "kl": 0.000124384619994089, + "learning_rate": 7.2000000000000005e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 81 + }, + { + "completion_length": 183.25, + "epoch": 0.010972835541281948, + "grad_norm": 0.0028198054060339928, + "kl": 0.000547908479347825, + "learning_rate": 7.28888888888889e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 82 + }, + { + "completion_length": 312.25, + "epoch": 0.011106650608858558, + "grad_norm": 0.27911630272865295, + "kl": 0.0005270105320960283, + "learning_rate": 7.377777777777778e-06, + "loss": 0.0, + "reward": 0.015999972820281982, + "reward_std": 1.7774159908294678, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.734000027179718, + "step": 83 + }, + { + "completion_length": 348.25, + "epoch": 0.011240465676435166, + "grad_norm": 0.0015268020797520876, + "kl": 0.00035788220702670515, + "learning_rate": 7.4666666666666675e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 84 + }, + { + "completion_length": 430.5, + "epoch": 0.011374280744011776, + "grad_norm": 0.13874612748622894, + "kl": 0.00037030907697044313, + "learning_rate": 7.555555555555556e-06, + "loss": 0.0, + "reward": -0.6794999837875366, + "reward_std": 1.3589999675750732, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.6794999837875366, + "step": 85 + }, + { + "completion_length": 264.5, + "epoch": 0.011508095811588384, + "grad_norm": 0.22207802534103394, + "kl": 0.00047897486365400255, + "learning_rate": 7.644444444444445e-06, + "loss": 0.0, + "reward": -0.3617500066757202, + "reward_std": 0.4231094419956207, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.3617500066757202, + "step": 86 + }, + { + "completion_length": 199.25, + "epoch": 0.011641910879164994, + "grad_norm": 0.26088446378707886, + "kl": 0.000934716546908021, + "learning_rate": 7.733333333333334e-06, + "loss": 0.0, + "reward": -0.17775000631809235, + "reward_std": 0.3555000126361847, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.17775000631809235, + "step": 87 + }, + { + "completion_length": 432.5, + "epoch": 0.011775725946741603, + "grad_norm": 0.22545722126960754, + "kl": 0.001022200332954526, + "learning_rate": 7.822222222222224e-06, + "loss": 0.0, + "reward": -0.10100000351667404, + "reward_std": 0.20200000703334808, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.10100000351667404, + "step": 88 + }, + { + "completion_length": 290.0, + "epoch": 0.011909541014318212, + "grad_norm": 0.0022929804399609566, + "kl": 0.0007066897815093398, + "learning_rate": 7.911111111111112e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 89 + }, + { + "completion_length": 306.5, + "epoch": 0.01204335608189482, + "grad_norm": 0.18963995575904846, + "kl": 0.0011429399019107223, + "learning_rate": 8.000000000000001e-06, + "loss": 0.0, + "reward": -0.3070000112056732, + "reward_std": 0.6140000224113464, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.3070000112056732, + "step": 90 + }, + { + "completion_length": 306.25, + "epoch": 0.01217717114947143, + "grad_norm": 0.32602232694625854, + "kl": 0.0012934200931340456, + "learning_rate": 8.08888888888889e-06, + "loss": 0.0001, + "reward": 0.021250000223517418, + "reward_std": 0.042500000447034836, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.021250000223517418, + "step": 91 + }, + { + "completion_length": 178.0, + "epoch": 0.01231098621704804, + "grad_norm": 0.32629159092903137, + "kl": 0.0017813891172409058, + "learning_rate": 8.177777777777779e-06, + "loss": 0.0001, + "reward": 0.6482499837875366, + "reward_std": 1.2964999675750732, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.023250000551342964, + "step": 92 + }, + { + "completion_length": 334.5, + "epoch": 0.012444801284624649, + "grad_norm": 0.0029060724191367626, + "kl": 0.0007846200605854392, + "learning_rate": 8.266666666666667e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 93 + }, + { + "completion_length": 389.5, + "epoch": 0.012578616352201259, + "grad_norm": 0.0021289298310875893, + "kl": 0.0007502713124267757, + "learning_rate": 8.355555555555556e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 94 + }, + { + "completion_length": 251.75, + "epoch": 0.012712431419777867, + "grad_norm": 0.011231260374188423, + "kl": 0.003988795448094606, + "learning_rate": 8.444444444444446e-06, + "loss": 0.0002, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 95 + }, + { + "completion_length": 246.25, + "epoch": 0.012846246487354477, + "grad_norm": 0.0036670551635324955, + "kl": 0.0013651195913553238, + "learning_rate": 8.533333333333335e-06, + "loss": 0.0001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 96 + }, + { + "completion_length": 338.25, + "epoch": 0.012980061554931085, + "grad_norm": 0.1569143831729889, + "kl": 0.0011384707177057862, + "learning_rate": 8.622222222222223e-06, + "loss": 0.0, + "reward": -0.36274999380111694, + "reward_std": 0.8109769225120544, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.36274999380111694, + "step": 97 + }, + { + "completion_length": 260.5, + "epoch": 0.013113876622507695, + "grad_norm": 0.2566815912723541, + "kl": 0.0018418811960145831, + "learning_rate": 8.711111111111111e-06, + "loss": 0.0001, + "reward": -0.10649999976158142, + "reward_std": 0.21299999952316284, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.10649999976158142, + "step": 98 + }, + { + "completion_length": 588.5, + "epoch": 0.013247691690084303, + "grad_norm": 0.3212381899356842, + "kl": 0.0008077286183834076, + "learning_rate": 8.8e-06, + "loss": 0.0, + "reward": -0.1547500044107437, + "reward_std": 0.30949997901916504, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.1547500044107437, + "step": 99 + }, + { + "completion_length": 202.5, + "epoch": 0.013381506757660913, + "grad_norm": 0.31029075384140015, + "kl": 0.0019261679844930768, + "learning_rate": 8.888888888888888e-06, + "loss": 0.0001, + "reward": 0.03125, + "reward_std": 0.0625, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.03125, + "step": 100 + }, + { + "completion_length": 422.0, + "epoch": 0.013515321825237521, + "grad_norm": 0.2212388813495636, + "kl": 0.0015066334744915366, + "learning_rate": 8.977777777777778e-06, + "loss": 0.0001, + "reward": 0.03324999660253525, + "reward_std": 0.27917900681495667, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.03324999660253525, + "step": 101 + }, + { + "completion_length": 405.25, + "epoch": 0.01364913689281413, + "grad_norm": 0.0010115457698702812, + "kl": 0.0004617396043613553, + "learning_rate": 9.066666666666667e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 102 + }, + { + "completion_length": 254.25, + "epoch": 0.01378295196039074, + "grad_norm": 0.0024686024989932775, + "kl": 0.0011656455462798476, + "learning_rate": 9.155555555555557e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 103 + }, + { + "completion_length": 270.75, + "epoch": 0.013916767027967349, + "grad_norm": 0.3148210048675537, + "kl": 0.0020634387619793415, + "learning_rate": 9.244444444444445e-06, + "loss": 0.0001, + "reward": -0.37025001645088196, + "reward_std": 0.9000205397605896, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.49525001645088196, + "step": 104 + }, + { + "completion_length": 447.25, + "epoch": 0.014050582095543959, + "grad_norm": 0.00280591519549489, + "kl": 0.0013167858123779297, + "learning_rate": 9.333333333333334e-06, + "loss": 0.0001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 105 + }, + { + "completion_length": 250.25, + "epoch": 0.014184397163120567, + "grad_norm": 1.272940993309021, + "kl": 0.010282534174621105, + "learning_rate": 9.422222222222222e-06, + "loss": 0.0004, + "reward": -0.38475000858306885, + "reward_std": 0.7695000171661377, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.38475000858306885, + "step": 106 + }, + { + "completion_length": 514.0, + "epoch": 0.014318212230697177, + "grad_norm": 0.003415808780118823, + "kl": 0.0018425981979817152, + "learning_rate": 9.511111111111112e-06, + "loss": 0.0001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 107 + }, + { + "completion_length": 378.25, + "epoch": 0.014452027298273785, + "grad_norm": 0.0043991003185510635, + "kl": 0.0022172422613948584, + "learning_rate": 9.600000000000001e-06, + "loss": 0.0001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 108 + }, + { + "completion_length": 384.0, + "epoch": 0.014585842365850395, + "grad_norm": 0.15473611652851105, + "kl": 0.003075521672144532, + "learning_rate": 9.688888888888889e-06, + "loss": 0.0001, + "reward": 0.03125, + "reward_std": 0.0625, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.03125, + "step": 109 + }, + { + "completion_length": 386.25, + "epoch": 0.014719657433427003, + "grad_norm": 0.18655884265899658, + "kl": 0.0021878289990127087, + "learning_rate": 9.777777777777779e-06, + "loss": 0.0001, + "reward": -0.14174999296665192, + "reward_std": 0.28349998593330383, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.14174999296665192, + "step": 110 + }, + { + "completion_length": 388.5, + "epoch": 0.014853472501003613, + "grad_norm": 0.005759637802839279, + "kl": 0.0032043028622865677, + "learning_rate": 9.866666666666668e-06, + "loss": 0.0001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 111 + }, + { + "completion_length": 368.0, + "epoch": 0.014987287568580221, + "grad_norm": 0.003658253001049161, + "kl": 0.0023007667623460293, + "learning_rate": 9.955555555555556e-06, + "loss": 0.0001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 112 + }, + { + "completion_length": 174.0, + "epoch": 0.015121102636156831, + "grad_norm": 0.01522353570908308, + "kl": 0.008999449200928211, + "learning_rate": 1.0044444444444446e-05, + "loss": 0.0004, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 113 + }, + { + "completion_length": 199.0, + "epoch": 0.015254917703733441, + "grad_norm": 0.2540055513381958, + "kl": 0.0061608292162418365, + "learning_rate": 1.0133333333333335e-05, + "loss": 0.0002, + "reward": 0.34974998235702515, + "reward_std": 0.24181587994098663, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.09974999725818634, + "step": 114 + }, + { + "completion_length": 261.0, + "epoch": 0.01538873277131005, + "grad_norm": 0.004695532377809286, + "kl": 0.0037373793311417103, + "learning_rate": 1.0222222222222223e-05, + "loss": 0.0001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 115 + }, + { + "completion_length": 460.75, + "epoch": 0.01552254783888666, + "grad_norm": 0.002644852502271533, + "kl": 0.0022986496333032846, + "learning_rate": 1.0311111111111113e-05, + "loss": 0.0001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 116 + }, + { + "completion_length": 199.75, + "epoch": 0.01565636290646327, + "grad_norm": 0.5485031604766846, + "kl": 0.0077914525754749775, + "learning_rate": 1.04e-05, + "loss": 0.0003, + "reward": 0.013749999925494194, + "reward_std": 0.11662010848522186, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.013749999925494194, + "step": 117 + }, + { + "completion_length": 221.5, + "epoch": 0.015790177974039876, + "grad_norm": 0.005294781178236008, + "kl": 0.004005097784101963, + "learning_rate": 1.048888888888889e-05, + "loss": 0.0002, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 118 + }, + { + "completion_length": 261.25, + "epoch": 0.015923993041616485, + "grad_norm": 0.005487331189215183, + "kl": 0.00457819364964962, + "learning_rate": 1.0577777777777778e-05, + "loss": 0.0002, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 119 + }, + { + "completion_length": 303.5, + "epoch": 0.016057808109193095, + "grad_norm": 0.18594208359718323, + "kl": 0.003819538513198495, + "learning_rate": 1.0666666666666667e-05, + "loss": 0.0002, + "reward": 0.23649999499320984, + "reward_std": 0.39350855350494385, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.11149999499320984, + "step": 120 + }, + { + "completion_length": 437.0, + "epoch": 0.016191623176769705, + "grad_norm": 0.003429972566664219, + "kl": 0.003270561108365655, + "learning_rate": 1.0755555555555557e-05, + "loss": 0.0001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 121 + }, + { + "completion_length": 225.75, + "epoch": 0.01632543824434631, + "grad_norm": 0.27138444781303406, + "kl": 0.008114825934171677, + "learning_rate": 1.0844444444444446e-05, + "loss": 0.0003, + "reward": 0.5557500123977661, + "reward_std": 1.1115000247955322, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.06925000250339508, + "step": 122 + }, + { + "completion_length": 228.75, + "epoch": 0.01645925331192292, + "grad_norm": 0.006109706126153469, + "kl": 0.0048021371476352215, + "learning_rate": 1.0933333333333334e-05, + "loss": 0.0002, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 123 + }, + { + "completion_length": 260.25, + "epoch": 0.01659306837949953, + "grad_norm": 0.22336947917938232, + "kl": 0.008044867776334286, + "learning_rate": 1.1022222222222224e-05, + "loss": 0.0003, + "reward": -0.14000000059604645, + "reward_std": 0.2800000011920929, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.14000000059604645, + "step": 124 + }, + { + "completion_length": 285.0, + "epoch": 0.01672688344707614, + "grad_norm": 0.1823970526456833, + "kl": 0.003958097659051418, + "learning_rate": 1.1111111111111113e-05, + "loss": 0.0002, + "reward": 0.03125, + "reward_std": 0.0625, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.03125, + "step": 125 + }, + { + "completion_length": 510.0, + "epoch": 0.01686069851465275, + "grad_norm": 0.21305835247039795, + "kl": 0.003483413252979517, + "learning_rate": 1.1200000000000001e-05, + "loss": 0.0001, + "reward": -0.11699999868869781, + "reward_std": 0.23399999737739563, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.11699999868869781, + "step": 126 + }, + { + "completion_length": 201.0, + "epoch": 0.016994513582229358, + "grad_norm": 0.00481327623128891, + "kl": 0.004589317366480827, + "learning_rate": 1.1288888888888889e-05, + "loss": 0.0002, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 127 + }, + { + "completion_length": 295.25, + "epoch": 0.017128328649805968, + "grad_norm": 0.019327977672219276, + "kl": 0.007599423639476299, + "learning_rate": 1.1377777777777779e-05, + "loss": 0.0003, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 128 + }, + { + "completion_length": 260.0, + "epoch": 0.017262143717382578, + "grad_norm": 0.25404590368270874, + "kl": 0.007972000166773796, + "learning_rate": 1.1466666666666668e-05, + "loss": 0.0003, + "reward": 0.5900000333786011, + "reward_std": 1.0515261888504028, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.034999996423721313, + "step": 129 + }, + { + "completion_length": 366.0, + "epoch": 0.017395958784959187, + "grad_norm": 0.2250823676586151, + "kl": 0.0046564992517232895, + "learning_rate": 1.1555555555555556e-05, + "loss": 0.0002, + "reward": -0.08275000005960464, + "reward_std": 0.25571519136428833, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.08275000005960464, + "step": 130 + }, + { + "completion_length": 200.25, + "epoch": 0.017529773852535794, + "grad_norm": 0.33885374665260315, + "kl": 0.010599642992019653, + "learning_rate": 1.1644444444444446e-05, + "loss": 0.0004, + "reward": 0.65625, + "reward_std": 1.3125, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.03125, + "step": 131 + }, + { + "completion_length": 203.25, + "epoch": 0.017663588920112404, + "grad_norm": 0.35624194145202637, + "kl": 0.00767766498029232, + "learning_rate": 1.1733333333333335e-05, + "loss": 0.0003, + "reward": 0.18774999678134918, + "reward_std": 0.2974899113178253, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.06274999678134918, + "step": 132 + }, + { + "completion_length": 225.0, + "epoch": 0.017797403987689014, + "grad_norm": 0.21870525181293488, + "kl": 0.005271477624773979, + "learning_rate": 1.1822222222222225e-05, + "loss": 0.0002, + "reward": 0.06875000149011612, + "reward_std": 0.13750000298023224, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.05624999850988388, + "step": 133 + }, + { + "completion_length": 299.25, + "epoch": 0.017931219055265624, + "grad_norm": 0.008637133985757828, + "kl": 0.00660554226487875, + "learning_rate": 1.191111111111111e-05, + "loss": 0.0003, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 134 + }, + { + "completion_length": 307.25, + "epoch": 0.018065034122842234, + "grad_norm": 0.32039323449134827, + "kl": 0.010449215769767761, + "learning_rate": 1.2e-05, + "loss": 0.0004, + "reward": 0.1067499965429306, + "reward_std": 0.12341629713773727, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.1067499965429306, + "step": 135 + }, + { + "completion_length": 323.0, + "epoch": 0.01819884919041884, + "grad_norm": 0.1738128513097763, + "kl": 0.00565626285970211, + "learning_rate": 1.208888888888889e-05, + "loss": 0.0002, + "reward": -0.37550002336502075, + "reward_std": 0.7510000467300415, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.5005000233650208, + "step": 136 + }, + { + "completion_length": 136.5, + "epoch": 0.01833266425799545, + "grad_norm": 0.5343514084815979, + "kl": 0.010712994262576103, + "learning_rate": 1.217777777777778e-05, + "loss": 0.0004, + "reward": 0.03849999979138374, + "reward_std": 0.07699999213218689, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.03849999979138374, + "step": 137 + }, + { + "completion_length": 393.5, + "epoch": 0.01846647932557206, + "grad_norm": 0.23455312848091125, + "kl": 0.0038801138289272785, + "learning_rate": 1.2266666666666667e-05, + "loss": 0.0002, + "reward": 0.125, + "reward_std": 0.25, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 138 + }, + { + "completion_length": 227.5, + "epoch": 0.01860029439314867, + "grad_norm": 0.011444750241935253, + "kl": 0.0084530059248209, + "learning_rate": 1.2355555555555557e-05, + "loss": 0.0003, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 139 + }, + { + "completion_length": 214.5, + "epoch": 0.018734109460725276, + "grad_norm": 0.004309752490371466, + "kl": 0.0035294159315526485, + "learning_rate": 1.2444444444444446e-05, + "loss": 0.0001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 140 + }, + { + "completion_length": 186.75, + "epoch": 0.018867924528301886, + "grad_norm": 0.30907976627349854, + "kl": 0.005284931510686874, + "learning_rate": 1.2533333333333336e-05, + "loss": 0.0002, + "reward": -0.02500000037252903, + "reward_std": 0.05000000074505806, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.02500000037252903, + "step": 141 + }, + { + "completion_length": 196.5, + "epoch": 0.019001739595878496, + "grad_norm": 0.482105016708374, + "kl": 0.006421719677746296, + "learning_rate": 1.2622222222222222e-05, + "loss": 0.0003, + "reward": 0.5855000019073486, + "reward_std": 1.3796979188919067, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.03949999809265137, + "step": 142 + }, + { + "completion_length": 147.0, + "epoch": 0.019135554663455106, + "grad_norm": 0.4996793866157532, + "kl": 0.008296657353639603, + "learning_rate": 1.2711111111111112e-05, + "loss": 0.0003, + "reward": -0.08299999684095383, + "reward_std": 0.340639591217041, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.20800000429153442, + "step": 143 + }, + { + "completion_length": 346.0, + "epoch": 0.019269369731031716, + "grad_norm": 0.2119971662759781, + "kl": 0.004949862137436867, + "learning_rate": 1.2800000000000001e-05, + "loss": 0.0002, + "reward": -0.26100000739097595, + "reward_std": 0.5668544769287109, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.26100000739097595, + "step": 144 + }, + { + "completion_length": 321.5, + "epoch": 0.019403184798608322, + "grad_norm": 0.2375519573688507, + "kl": 0.003724107053130865, + "learning_rate": 1.288888888888889e-05, + "loss": 0.0001, + "reward": 0.0794999971985817, + "reward_std": 0.49268415570259094, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.1705000102519989, + "step": 145 + }, + { + "completion_length": 268.25, + "epoch": 0.019536999866184932, + "grad_norm": 0.3193058967590332, + "kl": 0.0033836988732218742, + "learning_rate": 1.2977777777777779e-05, + "loss": 0.0001, + "reward": -0.11349999159574509, + "reward_std": 0.5760402083396912, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.23849999904632568, + "step": 146 + }, + { + "completion_length": 173.25, + "epoch": 0.019670814933761542, + "grad_norm": 0.26112160086631775, + "kl": 0.006745063699781895, + "learning_rate": 1.3066666666666668e-05, + "loss": 0.0003, + "reward": 1.871500015258789, + "reward_std": 1.0523637533187866, + "rewards/correctness_reward_func": 1.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.12849999964237213, + "step": 147 + }, + { + "completion_length": 261.75, + "epoch": 0.019804630001338152, + "grad_norm": 0.2869455814361572, + "kl": 0.008915035054087639, + "learning_rate": 1.3155555555555558e-05, + "loss": 0.0004, + "reward": 0.2330000102519989, + "reward_std": 0.4726901948451996, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.1080000028014183, + "step": 148 + }, + { + "completion_length": 401.25, + "epoch": 0.01993844506891476, + "grad_norm": 0.21081912517547607, + "kl": 0.0027889276389032602, + "learning_rate": 1.3244444444444447e-05, + "loss": 0.0001, + "reward": -0.21125000715255737, + "reward_std": 0.24462132155895233, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.21125000715255737, + "step": 149 + }, + { + "completion_length": 314.25, + "epoch": 0.02007226013649137, + "grad_norm": 0.19632020592689514, + "kl": 0.006448840722441673, + "learning_rate": 1.3333333333333333e-05, + "loss": 0.0003, + "reward": -0.29874998331069946, + "reward_std": 0.35072246193885803, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.29874998331069946, + "step": 150 + }, + { + "completion_length": 178.5, + "epoch": 0.020206075204067978, + "grad_norm": 0.43643537163734436, + "kl": 0.009863872081041336, + "learning_rate": 1.3422222222222223e-05, + "loss": 0.0004, + "reward": -0.001499999314546585, + "reward_std": 0.14289741218090057, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.001499999314546585, + "step": 151 + }, + { + "completion_length": 146.25, + "epoch": 0.020339890271644588, + "grad_norm": 0.32021835446357727, + "kl": 0.015754813328385353, + "learning_rate": 1.3511111111111112e-05, + "loss": 0.0006, + "reward": 0.11400000005960464, + "reward_std": 0.1562071293592453, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.11400000005960464, + "step": 152 + }, + { + "completion_length": 93.75, + "epoch": 0.020473705339221198, + "grad_norm": 0.531770646572113, + "kl": 0.015665946528315544, + "learning_rate": 1.3600000000000002e-05, + "loss": 0.0006, + "reward": 0.7382500171661377, + "reward_std": 1.344576358795166, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.1132500022649765, + "step": 153 + }, + { + "completion_length": 199.75, + "epoch": 0.020607520406797804, + "grad_norm": 0.46382462978363037, + "kl": 0.012057983316481113, + "learning_rate": 1.368888888888889e-05, + "loss": 0.0005, + "reward": 0.09950000047683716, + "reward_std": 0.14664356410503387, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.15049999952316284, + "step": 154 + }, + { + "completion_length": 378.0, + "epoch": 0.020741335474374414, + "grad_norm": 0.2936302125453949, + "kl": 0.00859862007200718, + "learning_rate": 1.377777777777778e-05, + "loss": 0.0003, + "reward": -0.07199999690055847, + "reward_std": 0.6001955270767212, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.19699999690055847, + "step": 155 + }, + { + "completion_length": 143.75, + "epoch": 0.020875150541951024, + "grad_norm": 0.36545681953430176, + "kl": 0.018919892609119415, + "learning_rate": 1.3866666666666669e-05, + "loss": 0.0008, + "reward": 0.2567500174045563, + "reward_std": 0.42568716406822205, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.13175000250339508, + "step": 156 + }, + { + "completion_length": 110.5, + "epoch": 0.021008965609527634, + "grad_norm": 0.5882317423820496, + "kl": 0.027741603553295135, + "learning_rate": 1.3955555555555558e-05, + "loss": 0.0011, + "reward": 0.17000000178813934, + "reward_std": 0.14358505606651306, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.17000000178813934, + "step": 157 + }, + { + "completion_length": 64.75, + "epoch": 0.02114278067710424, + "grad_norm": 1.1799137592315674, + "kl": 0.056555233895778656, + "learning_rate": 1.4044444444444445e-05, + "loss": 0.0023, + "reward": 0.38724997639656067, + "reward_std": 0.34333789348602295, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.26225000619888306, + "step": 158 + }, + { + "completion_length": 194.0, + "epoch": 0.02127659574468085, + "grad_norm": 0.30677542090415955, + "kl": 0.01423485018312931, + "learning_rate": 1.4133333333333334e-05, + "loss": 0.0006, + "reward": 0.10674998164176941, + "reward_std": 0.7794790267944336, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.1432499885559082, + "step": 159 + }, + { + "completion_length": 167.0, + "epoch": 0.02141041081225746, + "grad_norm": 0.42824897170066833, + "kl": 0.03500333055853844, + "learning_rate": 1.4222222222222224e-05, + "loss": 0.0014, + "reward": 0.3812499940395355, + "reward_std": 0.1884257048368454, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.13124999403953552, + "step": 160 + }, + { + "completion_length": 60.75, + "epoch": 0.02154422587983407, + "grad_norm": 0.5921115279197693, + "kl": 0.05752525106072426, + "learning_rate": 1.4311111111111111e-05, + "loss": 0.0023, + "reward": 0.2929999828338623, + "reward_std": 0.0557195246219635, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.2930000126361847, + "step": 161 + }, + { + "completion_length": 93.0, + "epoch": 0.021678040947410677, + "grad_norm": 0.6500325202941895, + "kl": 0.05201143026351929, + "learning_rate": 1.4400000000000001e-05, + "loss": 0.0021, + "reward": 0.17274999618530273, + "reward_std": 0.06422551721334457, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.17274999618530273, + "step": 162 + }, + { + "completion_length": 126.25, + "epoch": 0.021811856014987287, + "grad_norm": 0.44645747542381287, + "kl": 0.03292795643210411, + "learning_rate": 1.448888888888889e-05, + "loss": 0.0013, + "reward": 0.3970000147819519, + "reward_std": 0.22381241619586945, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.1469999998807907, + "step": 163 + }, + { + "completion_length": 79.75, + "epoch": 0.021945671082563897, + "grad_norm": 0.5969372391700745, + "kl": 0.06764098256826401, + "learning_rate": 1.457777777777778e-05, + "loss": 0.0027, + "reward": 0.1627500057220459, + "reward_std": 0.09626135975122452, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.1627500057220459, + "step": 164 + }, + { + "completion_length": 245.75, + "epoch": 0.022079486150140507, + "grad_norm": 0.42619219422340393, + "kl": 0.031476035714149475, + "learning_rate": 1.4666666666666666e-05, + "loss": 0.0013, + "reward": 0.5945000052452087, + "reward_std": 0.4530264735221863, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.21949999034404755, + "step": 165 + }, + { + "completion_length": 106.5, + "epoch": 0.022213301217717116, + "grad_norm": 0.6185816526412964, + "kl": 0.07715773582458496, + "learning_rate": 1.4755555555555556e-05, + "loss": 0.0031, + "reward": 0.5134999752044678, + "reward_std": 0.33077535033226013, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.13850000500679016, + "step": 166 + }, + { + "completion_length": 80.0, + "epoch": 0.022347116285293723, + "grad_norm": 0.5252732038497925, + "kl": 0.09305495023727417, + "learning_rate": 1.4844444444444445e-05, + "loss": 0.0037, + "reward": 0.23899999260902405, + "reward_std": 0.10646440088748932, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.23899999260902405, + "step": 167 + }, + { + "completion_length": 279.0, + "epoch": 0.022480931352870333, + "grad_norm": 0.260774165391922, + "kl": 0.04496491700410843, + "learning_rate": 1.4933333333333335e-05, + "loss": 0.0018, + "reward": 0.11250001192092896, + "reward_std": 1.9712051153182983, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -1.1374999284744263, + "step": 168 + }, + { + "completion_length": 108.25, + "epoch": 0.022614746420446943, + "grad_norm": 0.4881371259689331, + "kl": 0.05013870447874069, + "learning_rate": 1.5022222222222223e-05, + "loss": 0.002, + "reward": 2.03725004196167, + "reward_std": 1.0539665222167969, + "rewards/correctness_reward_func": 1.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.03725000470876694, + "step": 169 + }, + { + "completion_length": 99.25, + "epoch": 0.022748561488023553, + "grad_norm": 0.5610305070877075, + "kl": 0.043805379420518875, + "learning_rate": 1.5111111111111112e-05, + "loss": 0.0018, + "reward": 1.1447501182556152, + "reward_std": 1.0834035873413086, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.14474999904632568, + "step": 170 + }, + { + "completion_length": 94.75, + "epoch": 0.02288237655560016, + "grad_norm": 0.4598788917064667, + "kl": 0.0532003678381443, + "learning_rate": 1.5200000000000002e-05, + "loss": 0.0021, + "reward": 0.7122499942779541, + "reward_std": 0.19165311753749847, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.2122499942779541, + "step": 171 + }, + { + "completion_length": 137.25, + "epoch": 0.02301619162317677, + "grad_norm": 0.34965983033180237, + "kl": 0.03090197592973709, + "learning_rate": 1.528888888888889e-05, + "loss": 0.0012, + "reward": 0.4805000126361847, + "reward_std": 0.415036141872406, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.10550001263618469, + "step": 172 + }, + { + "completion_length": 207.0, + "epoch": 0.02315000669075338, + "grad_norm": 0.5801133513450623, + "kl": 0.02816297486424446, + "learning_rate": 1.537777777777778e-05, + "loss": 0.0011, + "reward": 0.07525002956390381, + "reward_std": 0.49743297696113586, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.2997499704360962, + "step": 173 + }, + { + "completion_length": 65.75, + "epoch": 0.02328382175832999, + "grad_norm": 0.6282637715339661, + "kl": 0.11807440221309662, + "learning_rate": 1.546666666666667e-05, + "loss": 0.0047, + "reward": 0.6984999775886536, + "reward_std": 0.19891957938671112, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.32350003719329834, + "step": 174 + }, + { + "completion_length": 70.5, + "epoch": 0.0234176368259066, + "grad_norm": 0.6953238248825073, + "kl": 0.069422647356987, + "learning_rate": 1.555555555555556e-05, + "loss": 0.0028, + "reward": 0.6110000014305115, + "reward_std": 0.4198293089866638, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.23600000143051147, + "step": 175 + }, + { + "completion_length": 86.5, + "epoch": 0.023551451893483205, + "grad_norm": 1.0817539691925049, + "kl": 0.08881973475217819, + "learning_rate": 1.5644444444444448e-05, + "loss": 0.0036, + "reward": 0.6867499947547913, + "reward_std": 0.059885308146476746, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.18674999475479126, + "step": 176 + }, + { + "completion_length": 122.5, + "epoch": 0.023685266961059815, + "grad_norm": 0.3636866509914398, + "kl": 0.056608811020851135, + "learning_rate": 1.5733333333333334e-05, + "loss": 0.0023, + "reward": 1.746999979019165, + "reward_std": 1.2025237083435059, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.24700000882148743, + "step": 177 + }, + { + "completion_length": 90.25, + "epoch": 0.023819082028636425, + "grad_norm": 0.7407636642456055, + "kl": 0.07338247448205948, + "learning_rate": 1.5822222222222224e-05, + "loss": 0.0029, + "reward": 0.5740000009536743, + "reward_std": 0.23982912302017212, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.19900000095367432, + "step": 178 + }, + { + "completion_length": 70.75, + "epoch": 0.023952897096213035, + "grad_norm": 0.731931746006012, + "kl": 0.08679507672786713, + "learning_rate": 1.5911111111111113e-05, + "loss": 0.0035, + "reward": 1.2517499923706055, + "reward_std": 0.9867871999740601, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.25174999237060547, + "step": 179 + }, + { + "completion_length": 87.75, + "epoch": 0.02408671216378964, + "grad_norm": 0.40641459822654724, + "kl": 0.05429335683584213, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.0022, + "reward": 1.221750020980835, + "reward_std": 1.0200151205062866, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.22175000607967377, + "step": 180 + }, + { + "completion_length": 98.5, + "epoch": 0.02422052723136625, + "grad_norm": 0.3797653615474701, + "kl": 0.13556654751300812, + "learning_rate": 1.608888888888889e-05, + "loss": 0.0054, + "reward": 0.9449999928474426, + "reward_std": 1.1976945400238037, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.19500002264976501, + "step": 181 + }, + { + "completion_length": 93.5, + "epoch": 0.02435434229894286, + "grad_norm": 0.3987956643104553, + "kl": 0.06347194314002991, + "learning_rate": 1.617777777777778e-05, + "loss": 0.0025, + "reward": 0.5089999437332153, + "reward_std": 0.30833208560943604, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.1339999884366989, + "step": 182 + }, + { + "completion_length": 167.75, + "epoch": 0.02448815736651947, + "grad_norm": 0.3698064982891083, + "kl": 0.034620750695466995, + "learning_rate": 1.6266666666666668e-05, + "loss": 0.0014, + "reward": 0.3659999966621399, + "reward_std": 0.4141714572906494, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.1340000033378601, + "step": 183 + }, + { + "completion_length": 59.5, + "epoch": 0.02462197243409608, + "grad_norm": 1.24988853931427, + "kl": 0.14011864364147186, + "learning_rate": 1.6355555555555557e-05, + "loss": 0.0056, + "reward": 1.0192499160766602, + "reward_std": 1.1953402757644653, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.26924997568130493, + "step": 184 + }, + { + "completion_length": 80.25, + "epoch": 0.024755787501672687, + "grad_norm": 0.5655855536460876, + "kl": 0.05728255584836006, + "learning_rate": 1.6444444444444444e-05, + "loss": 0.0023, + "reward": 0.8680000305175781, + "reward_std": 1.225602626800537, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.24300000071525574, + "step": 185 + }, + { + "completion_length": 86.75, + "epoch": 0.024889602569249297, + "grad_norm": 0.7822229266166687, + "kl": 0.08518431335687637, + "learning_rate": 1.6533333333333333e-05, + "loss": 0.0034, + "reward": 0.5862500071525574, + "reward_std": 0.19938969612121582, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.08624999970197678, + "step": 186 + }, + { + "completion_length": 67.75, + "epoch": 0.025023417636825907, + "grad_norm": 0.5667008757591248, + "kl": 0.06875781714916229, + "learning_rate": 1.6622222222222223e-05, + "loss": 0.0028, + "reward": 1.2515000104904175, + "reward_std": 0.9990040063858032, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.2515000104904175, + "step": 187 + }, + { + "completion_length": 93.0, + "epoch": 0.025157232704402517, + "grad_norm": 0.456777960062027, + "kl": 0.06855905055999756, + "learning_rate": 1.6711111111111112e-05, + "loss": 0.0027, + "reward": 0.9950000047683716, + "reward_std": 1.1864756345748901, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.24500000476837158, + "step": 188 + }, + { + "completion_length": 103.5, + "epoch": 0.025291047771979124, + "grad_norm": 0.6969872713088989, + "kl": 0.09152114391326904, + "learning_rate": 1.6800000000000002e-05, + "loss": 0.0037, + "reward": 1.597749948501587, + "reward_std": 1.3118852376937866, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.09775000065565109, + "step": 189 + }, + { + "completion_length": 59.5, + "epoch": 0.025424862839555733, + "grad_norm": 0.9019510746002197, + "kl": 0.2068648785352707, + "learning_rate": 1.688888888888889e-05, + "loss": 0.0083, + "reward": 0.5332499742507935, + "reward_std": 0.2600556015968323, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.28325000405311584, + "step": 190 + }, + { + "completion_length": 121.75, + "epoch": 0.025558677907132343, + "grad_norm": 0.4399167001247406, + "kl": 0.09197084605693817, + "learning_rate": 1.697777777777778e-05, + "loss": 0.0037, + "reward": 0.5910000205039978, + "reward_std": 0.3367471992969513, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.09100000560283661, + "step": 191 + }, + { + "completion_length": 57.75, + "epoch": 0.025692492974708953, + "grad_norm": 0.47419285774230957, + "kl": 0.12339113652706146, + "learning_rate": 1.706666666666667e-05, + "loss": 0.0049, + "reward": 0.2979999780654907, + "reward_std": 0.032321300357580185, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.2979999780654907, + "step": 192 + }, + { + "completion_length": 73.0, + "epoch": 0.02582630804228556, + "grad_norm": 0.5090747475624084, + "kl": 0.11672109365463257, + "learning_rate": 1.7155555555555557e-05, + "loss": 0.0047, + "reward": 1.25, + "reward_std": 1.0, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.25, + "step": 193 + }, + { + "completion_length": 78.0, + "epoch": 0.02596012310986217, + "grad_norm": 0.5393621921539307, + "kl": 0.07206133008003235, + "learning_rate": 1.7244444444444446e-05, + "loss": 0.0029, + "reward": 2.25, + "reward_std": 1.0, + "rewards/correctness_reward_func": 1.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.25, + "step": 194 + }, + { + "completion_length": 76.25, + "epoch": 0.02609393817743878, + "grad_norm": 0.542004406452179, + "kl": 0.12011324614286423, + "learning_rate": 1.7333333333333336e-05, + "loss": 0.0048, + "reward": 0.625, + "reward_std": 0.25, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.25, + "step": 195 + }, + { + "completion_length": 106.0, + "epoch": 0.02622775324501539, + "grad_norm": 0.43443137407302856, + "kl": 0.053334061056375504, + "learning_rate": 1.7422222222222222e-05, + "loss": 0.0021, + "reward": 2.25, + "reward_std": 1.0, + "rewards/correctness_reward_func": 1.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.25, + "step": 196 + }, + { + "completion_length": 70.25, + "epoch": 0.026361568312592, + "grad_norm": 0.5421243906021118, + "kl": 0.14290496706962585, + "learning_rate": 1.751111111111111e-05, + "loss": 0.0057, + "reward": 1.1717499494552612, + "reward_std": 0.9182985424995422, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.1717499941587448, + "step": 197 + }, + { + "completion_length": 50.0, + "epoch": 0.026495383380168606, + "grad_norm": 0.5998945832252502, + "kl": 0.16523699462413788, + "learning_rate": 1.76e-05, + "loss": 0.0066, + "reward": 1.125, + "reward_std": 1.108677864074707, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.25, + "step": 198 + }, + { + "completion_length": 83.5, + "epoch": 0.026629198447745216, + "grad_norm": 0.08789081871509552, + "kl": 0.14140745997428894, + "learning_rate": 1.768888888888889e-05, + "loss": 0.0057, + "reward": 0.75, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.25, + "step": 199 + }, + { + "completion_length": 94.0, + "epoch": 0.026763013515321826, + "grad_norm": 0.8682589530944824, + "kl": 0.24527359008789062, + "learning_rate": 1.7777777777777777e-05, + "loss": 0.0098, + "reward": 0.715499997138977, + "reward_std": 0.15306098759174347, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.21549999713897705, + "step": 200 + }, + { + "completion_length": 85.75, + "epoch": 0.026896828582898435, + "grad_norm": 0.5800766944885254, + "kl": 0.1065763458609581, + "learning_rate": 1.7866666666666666e-05, + "loss": 0.0043, + "reward": 0.6587499976158142, + "reward_std": 0.11285203695297241, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.1587499976158142, + "step": 201 + }, + { + "completion_length": 72.0, + "epoch": 0.027030643650475042, + "grad_norm": 0.6035862565040588, + "kl": 0.11155687272548676, + "learning_rate": 1.7955555555555556e-05, + "loss": 0.0045, + "reward": 1.316499948501587, + "reward_std": 0.9577993750572205, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.3165000081062317, + "step": 202 + }, + { + "completion_length": 82.75, + "epoch": 0.027164458718051652, + "grad_norm": 0.44579821825027466, + "kl": 0.0726730078458786, + "learning_rate": 1.8044444444444445e-05, + "loss": 0.0029, + "reward": 1.25, + "reward_std": 1.0, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.25, + "step": 203 + }, + { + "completion_length": 69.0, + "epoch": 0.02729827378562826, + "grad_norm": 0.6042316555976868, + "kl": 0.10065613687038422, + "learning_rate": 1.8133333333333335e-05, + "loss": 0.004, + "reward": 0.7465000152587891, + "reward_std": 0.006999989040195942, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.24650000035762787, + "step": 204 + }, + { + "completion_length": 53.75, + "epoch": 0.02743208885320487, + "grad_norm": 0.9847255945205688, + "kl": 0.16128680109977722, + "learning_rate": 1.8222222222222224e-05, + "loss": 0.0065, + "reward": 0.2667500078678131, + "reward_std": 0.033500004559755325, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.2667500078678131, + "step": 205 + }, + { + "completion_length": 59.5, + "epoch": 0.02756590392078148, + "grad_norm": 0.642667829990387, + "kl": 0.1901407241821289, + "learning_rate": 1.8311111111111114e-05, + "loss": 0.0076, + "reward": 0.503000020980835, + "reward_std": 0.29218029975891113, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.2529999911785126, + "step": 206 + }, + { + "completion_length": 76.0, + "epoch": 0.027699718988358088, + "grad_norm": 1.0304243564605713, + "kl": 0.36415040493011475, + "learning_rate": 1.8400000000000003e-05, + "loss": 0.0146, + "reward": 2.25, + "reward_std": 1.0, + "rewards/correctness_reward_func": 1.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.25, + "step": 207 + }, + { + "completion_length": 67.0, + "epoch": 0.027833534055934698, + "grad_norm": 0.7824116945266724, + "kl": 0.35757210850715637, + "learning_rate": 1.848888888888889e-05, + "loss": 0.0143, + "reward": 1.444000005722046, + "reward_std": 1.5823231935501099, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.1940000057220459, + "step": 208 + }, + { + "completion_length": 66.0, + "epoch": 0.027967349123511308, + "grad_norm": 0.6381920576095581, + "kl": 0.16021761298179626, + "learning_rate": 1.857777777777778e-05, + "loss": 0.0064, + "reward": 0.7674999833106995, + "reward_std": 0.03348132595419884, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.26749998331069946, + "step": 209 + }, + { + "completion_length": 51.5, + "epoch": 0.028101164191087918, + "grad_norm": 0.029546145349740982, + "kl": 0.1807194948196411, + "learning_rate": 1.866666666666667e-05, + "loss": 0.0072, + "reward": 0.75, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.25, + "step": 210 + }, + { + "completion_length": 124.25, + "epoch": 0.028234979258664524, + "grad_norm": 1.192435622215271, + "kl": 0.16933509707450867, + "learning_rate": 1.8755555555555558e-05, + "loss": 0.0068, + "reward": 0.6417499780654907, + "reward_std": 0.12795408070087433, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.1417500078678131, + "step": 211 + }, + { + "completion_length": 61.75, + "epoch": 0.028368794326241134, + "grad_norm": 0.4982728660106659, + "kl": 0.10138452053070068, + "learning_rate": 1.8844444444444444e-05, + "loss": 0.0041, + "reward": 0.746999979019165, + "reward_std": 0.06240728497505188, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.24700000882148743, + "step": 212 + }, + { + "completion_length": 80.25, + "epoch": 0.028502609393817744, + "grad_norm": 0.4456157386302948, + "kl": 0.10433340817689896, + "learning_rate": 1.8933333333333334e-05, + "loss": 0.0042, + "reward": 1.25, + "reward_std": 1.0, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.25, + "step": 213 + }, + { + "completion_length": 146.25, + "epoch": 0.028636424461394354, + "grad_norm": 0.24999132752418518, + "kl": 0.06005653738975525, + "learning_rate": 1.9022222222222223e-05, + "loss": 0.0024, + "reward": 0.574999988079071, + "reward_std": 0.22246196866035461, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.20000000298023224, + "step": 214 + }, + { + "completion_length": 66.25, + "epoch": 0.028770239528970964, + "grad_norm": 0.7781875133514404, + "kl": 0.16475608944892883, + "learning_rate": 1.9111111111111113e-05, + "loss": 0.0066, + "reward": 0.746999979019165, + "reward_std": 0.10004331171512604, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.24700000882148743, + "step": 215 + }, + { + "completion_length": 39.75, + "epoch": 0.02890405459654757, + "grad_norm": 0.7784989476203918, + "kl": 0.14634883403778076, + "learning_rate": 1.9200000000000003e-05, + "loss": 0.0059, + "reward": 1.7787500619888306, + "reward_std": 1.170571208000183, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.2787500023841858, + "step": 216 + }, + { + "completion_length": 72.25, + "epoch": 0.02903786966412418, + "grad_norm": 0.4837649464607239, + "kl": 0.09058363735675812, + "learning_rate": 1.928888888888889e-05, + "loss": 0.0036, + "reward": 0.27175000309944153, + "reward_std": 0.3053712248802185, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.14675000309944153, + "step": 217 + }, + { + "completion_length": 80.25, + "epoch": 0.02917168473170079, + "grad_norm": 0.9711887836456299, + "kl": 0.07635970413684845, + "learning_rate": 1.9377777777777778e-05, + "loss": 0.0031, + "reward": 1.7442500591278076, + "reward_std": 1.0721337795257568, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.24424999952316284, + "step": 218 + }, + { + "completion_length": 70.5, + "epoch": 0.0293054997992774, + "grad_norm": 0.42600736021995544, + "kl": 0.06140553951263428, + "learning_rate": 1.9466666666666668e-05, + "loss": 0.0025, + "reward": 0.7387499809265137, + "reward_std": 0.05771408975124359, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.23874998092651367, + "step": 219 + }, + { + "completion_length": 217.5, + "epoch": 0.029439314866854006, + "grad_norm": 0.5537015199661255, + "kl": 0.05942856892943382, + "learning_rate": 1.9555555555555557e-05, + "loss": 0.0024, + "reward": 0.3062500059604645, + "reward_std": 0.3994557559490204, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.19374999403953552, + "step": 220 + }, + { + "completion_length": 135.75, + "epoch": 0.029573129934430616, + "grad_norm": 0.29336312413215637, + "kl": 0.052648480981588364, + "learning_rate": 1.9644444444444447e-05, + "loss": 0.0021, + "reward": 0.5582500100135803, + "reward_std": 0.29113730788230896, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.18325001001358032, + "step": 221 + }, + { + "completion_length": 62.0, + "epoch": 0.029706945002007226, + "grad_norm": 0.6704282760620117, + "kl": 0.09709338843822479, + "learning_rate": 1.9733333333333336e-05, + "loss": 0.0039, + "reward": 0.7619999647140503, + "reward_std": 0.038531359285116196, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.2619999945163727, + "step": 222 + }, + { + "completion_length": 116.5, + "epoch": 0.029840760069583836, + "grad_norm": 0.2809985876083374, + "kl": 0.11135183274745941, + "learning_rate": 1.9822222222222226e-05, + "loss": 0.0045, + "reward": 0.5497499704360962, + "reward_std": 0.36881014704704285, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.17475000023841858, + "step": 223 + }, + { + "completion_length": 60.5, + "epoch": 0.029974575137160443, + "grad_norm": 0.6438567638397217, + "kl": 0.133337140083313, + "learning_rate": 1.9911111111111112e-05, + "loss": 0.0053, + "reward": 1.3017499446868896, + "reward_std": 0.9685468077659607, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.3017500042915344, + "step": 224 + }, + { + "completion_length": 74.0, + "epoch": 0.030108390204737052, + "grad_norm": 0.4634908139705658, + "kl": 0.08658132702112198, + "learning_rate": 2e-05, + "loss": 0.0035, + "reward": 1.0735000371932983, + "reward_std": 1.0860450267791748, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.19850000739097595, + "step": 225 + }, + { + "completion_length": 61.5, + "epoch": 0.030242205272313662, + "grad_norm": 0.6003581285476685, + "kl": 0.12208070605993271, + "learning_rate": 1.9999999060637166e-05, + "loss": 0.0049, + "reward": 0.7789999842643738, + "reward_std": 0.06427544355392456, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.2789999842643738, + "step": 226 + }, + { + "completion_length": 73.5, + "epoch": 0.030376020339890272, + "grad_norm": 0.5231612920761108, + "kl": 0.15073804557323456, + "learning_rate": 1.9999996242548837e-05, + "loss": 0.006, + "reward": 1.16225004196167, + "reward_std": 1.0848207473754883, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.28724998235702515, + "step": 227 + }, + { + "completion_length": 72.5, + "epoch": 0.030509835407466882, + "grad_norm": 0.4123309552669525, + "kl": 0.07140478491783142, + "learning_rate": 1.999999154573555e-05, + "loss": 0.0029, + "reward": 0.7667500376701355, + "reward_std": 0.04211394116282463, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.2667500078678131, + "step": 228 + }, + { + "completion_length": 88.5, + "epoch": 0.03064365047504349, + "grad_norm": 0.5252578854560852, + "kl": 0.03713103383779526, + "learning_rate": 1.9999984970198176e-05, + "loss": 0.0015, + "reward": 1.5755000114440918, + "reward_std": 1.2587233781814575, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.2004999965429306, + "step": 229 + }, + { + "completion_length": 145.0, + "epoch": 0.0307774655426201, + "grad_norm": 0.4767831563949585, + "kl": 0.07187433540821075, + "learning_rate": 1.999997651593796e-05, + "loss": 0.0029, + "reward": 0.4542499780654907, + "reward_std": 0.4314698874950409, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.045750007033348083, + "step": 230 + }, + { + "completion_length": 102.75, + "epoch": 0.03091128061019671, + "grad_norm": 0.4800109267234802, + "kl": 0.07859396934509277, + "learning_rate": 1.9999966182956486e-05, + "loss": 0.0031, + "reward": 0.6054999828338623, + "reward_std": 0.21153488755226135, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.1054999977350235, + "step": 231 + }, + { + "completion_length": 190.25, + "epoch": 0.03104509567777332, + "grad_norm": 0.20692415535449982, + "kl": 0.04673830792307854, + "learning_rate": 1.9999953971255692e-05, + "loss": 0.0019, + "reward": 0.15949998795986176, + "reward_std": 0.31504231691360474, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.125, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.03450000286102295, + "step": 232 + }, + { + "completion_length": 68.0, + "epoch": 0.031178910745349925, + "grad_norm": 0.5570704936981201, + "kl": 0.08012831211090088, + "learning_rate": 1.999993988083788e-05, + "loss": 0.0032, + "reward": 2.252500057220459, + "reward_std": 0.95698082447052, + "rewards/correctness_reward_func": 1.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.2524999976158142, + "step": 233 + }, + { + "completion_length": 131.5, + "epoch": 0.03131272581292654, + "grad_norm": 0.4668695628643036, + "kl": 0.0665660873055458, + "learning_rate": 1.9999923911705693e-05, + "loss": 0.0027, + "reward": 1.5532499551773071, + "reward_std": 1.3141237497329712, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.17824998497962952, + "step": 234 + }, + { + "completion_length": 73.25, + "epoch": 0.031446540880503145, + "grad_norm": 0.5236088633537292, + "kl": 0.14625723659992218, + "learning_rate": 1.9999906063862128e-05, + "loss": 0.0059, + "reward": 1.7309999465942383, + "reward_std": 1.0541173219680786, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.23100000619888306, + "step": 235 + }, + { + "completion_length": 114.25, + "epoch": 0.03158035594807975, + "grad_norm": 0.45700567960739136, + "kl": 0.08767452090978622, + "learning_rate": 1.9999886337310546e-05, + "loss": 0.0035, + "reward": 0.4410000145435333, + "reward_std": 0.4759082794189453, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.06599999964237213, + "step": 236 + }, + { + "completion_length": 208.0, + "epoch": 0.031714171015656364, + "grad_norm": 0.45323336124420166, + "kl": 0.0659441202878952, + "learning_rate": 1.999986473205465e-05, + "loss": 0.0026, + "reward": 0.2602500021457672, + "reward_std": 0.6481220722198486, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.2397499978542328, + "step": 237 + }, + { + "completion_length": 140.25, + "epoch": 0.03184798608323297, + "grad_norm": 0.4077318608760834, + "kl": 0.05185340344905853, + "learning_rate": 1.999984124809849e-05, + "loss": 0.0021, + "reward": 1.4795000553131104, + "reward_std": 1.379389762878418, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.10450000315904617, + "step": 238 + }, + { + "completion_length": 70.0, + "epoch": 0.031981801150809584, + "grad_norm": 0.926057755947113, + "kl": 0.11616024374961853, + "learning_rate": 1.9999815885446497e-05, + "loss": 0.0046, + "reward": 0.9787499904632568, + "reward_std": 0.3524896502494812, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.125, + "rewards/xmlcount_reward_func": 0.35374999046325684, + "step": 239 + }, + { + "completion_length": 293.5, + "epoch": 0.03211561621838619, + "grad_norm": 0.394625723361969, + "kl": 0.017029505223035812, + "learning_rate": 1.9999788644103418e-05, + "loss": 0.0007, + "reward": -0.1302500218153, + "reward_std": 0.6652750968933105, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.6302499771118164, + "step": 240 + }, + { + "completion_length": 135.0, + "epoch": 0.0322494312859628, + "grad_norm": 0.2847868502140045, + "kl": 0.09572663903236389, + "learning_rate": 1.9999759524074374e-05, + "loss": 0.0038, + "reward": 0.7019999623298645, + "reward_std": 0.22655829787254333, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.2019999921321869, + "step": 241 + }, + { + "completion_length": 112.25, + "epoch": 0.03238324635353941, + "grad_norm": 0.5767569541931152, + "kl": 0.12632526457309723, + "learning_rate": 1.9999728525364848e-05, + "loss": 0.0051, + "reward": 0.6430000066757202, + "reward_std": 0.35005998611450195, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.14300000667572021, + "step": 242 + }, + { + "completion_length": 126.5, + "epoch": 0.03251706142111602, + "grad_norm": 0.3967666029930115, + "kl": 0.11589077860116959, + "learning_rate": 1.999969564798065e-05, + "loss": 0.0046, + "reward": 1.4282499551773071, + "reward_std": 0.553694486618042, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.25, + "rewards/xmlcount_reward_func": 0.1782499998807907, + "step": 243 + }, + { + "completion_length": 134.0, + "epoch": 0.03265087648869262, + "grad_norm": 0.38813862204551697, + "kl": 0.06340669095516205, + "learning_rate": 1.999966089192796e-05, + "loss": 0.0025, + "reward": 1.2892500162124634, + "reward_std": 1.5317614078521729, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.125, + "rewards/xmlcount_reward_func": 0.2892500162124634, + "step": 244 + }, + { + "completion_length": 109.75, + "epoch": 0.03278469155626924, + "grad_norm": 0.42066898941993713, + "kl": 0.12392938882112503, + "learning_rate": 1.9999624257213318e-05, + "loss": 0.005, + "reward": 1.7067500352859497, + "reward_std": 1.3346679210662842, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.20675000548362732, + "step": 245 + }, + { + "completion_length": 102.0, + "epoch": 0.03291850662384584, + "grad_norm": 0.704933762550354, + "kl": 0.12700574100017548, + "learning_rate": 1.9999585743843592e-05, + "loss": 0.0051, + "reward": 0.6620000004768372, + "reward_std": 0.6284016370773315, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.125, + "rewards/xmlcount_reward_func": 0.16200000047683716, + "step": 246 + }, + { + "completion_length": 65.75, + "epoch": 0.03305232169142246, + "grad_norm": 0.557697057723999, + "kl": 0.16324672102928162, + "learning_rate": 1.9999545351826028e-05, + "loss": 0.0065, + "reward": 1.4605000019073486, + "reward_std": 1.3599356412887573, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.125, + "rewards/xmlcount_reward_func": 0.335500031709671, + "step": 247 + }, + { + "completion_length": 50.25, + "epoch": 0.03318613675899906, + "grad_norm": 0.8064447641372681, + "kl": 0.2082303762435913, + "learning_rate": 1.9999503081168205e-05, + "loss": 0.0083, + "reward": 0.7957500219345093, + "reward_std": 0.09577882289886475, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.2957499921321869, + "step": 248 + }, + { + "completion_length": 92.0, + "epoch": 0.03331995182657567, + "grad_norm": 0.8607310652732849, + "kl": 0.11370626091957092, + "learning_rate": 1.999945893187807e-05, + "loss": 0.0045, + "reward": 1.3242499828338623, + "reward_std": 0.351500004529953, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.375, + "rewards/xmlcount_reward_func": 0.4492499828338623, + "step": 249 + }, + { + "completion_length": 120.5, + "epoch": 0.03345376689415228, + "grad_norm": 0.46249887347221375, + "kl": 0.09916168451309204, + "learning_rate": 1.9999412903963925e-05, + "loss": 0.004, + "reward": 0.6180000305175781, + "reward_std": 0.44016435742378235, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.24300000071525574, + "step": 250 + }, + { + "completion_length": 121.75, + "epoch": 0.03358758196172889, + "grad_norm": 0.7673288583755493, + "kl": 0.14884337782859802, + "learning_rate": 1.9999364997434406e-05, + "loss": 0.006, + "reward": 0.843999981880188, + "reward_std": 1.0066499710083008, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.25, + "rewards/xmlcount_reward_func": 0.09400001168251038, + "step": 251 + }, + { + "completion_length": 67.5, + "epoch": 0.0337213970293055, + "grad_norm": 0.6919443607330322, + "kl": 0.20414642989635468, + "learning_rate": 1.9999315212298516e-05, + "loss": 0.0082, + "reward": 2.0, + "reward_std": 1.0, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.5, + "rewards/xmlcount_reward_func": 0.5, + "step": 252 + }, + { + "completion_length": 105.0, + "epoch": 0.03385521209688211, + "grad_norm": 0.4287322461605072, + "kl": 0.16063739359378815, + "learning_rate": 1.999926354856561e-05, + "loss": 0.0064, + "reward": 1.5625, + "reward_std": 0.9213893413543701, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.25, + "rewards/xmlcount_reward_func": 0.3125, + "step": 253 + }, + { + "completion_length": 109.75, + "epoch": 0.033989027164458716, + "grad_norm": 0.5036614537239075, + "kl": 0.10087580978870392, + "learning_rate": 1.9999210006245395e-05, + "loss": 0.004, + "reward": 1.224250078201294, + "reward_std": 0.850592851638794, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.22424998879432678, + "step": 254 + }, + { + "completion_length": 102.5, + "epoch": 0.03412284223203533, + "grad_norm": 0.4636721909046173, + "kl": 0.10581967979669571, + "learning_rate": 1.9999154585347926e-05, + "loss": 0.0042, + "reward": 1.4709999561309814, + "reward_std": 1.0897164344787598, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.125, + "rewards/xmlcount_reward_func": 0.3460000157356262, + "step": 255 + }, + { + "completion_length": 72.25, + "epoch": 0.034256657299611935, + "grad_norm": 0.03021317906677723, + "kl": 0.1458757519721985, + "learning_rate": 1.999909728588362e-05, + "loss": 0.0058, + "reward": 1.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.5, + "rewards/xmlcount_reward_func": 0.5, + "step": 256 + }, + { + "completion_length": 117.25, + "epoch": 0.03439047236718855, + "grad_norm": 0.4804219901561737, + "kl": 0.1489054560661316, + "learning_rate": 1.999903810786324e-05, + "loss": 0.006, + "reward": 1.563499927520752, + "reward_std": 0.5171675682067871, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.25, + "rewards/xmlcount_reward_func": 0.31349998712539673, + "step": 257 + }, + { + "completion_length": 132.0, + "epoch": 0.034524287434765155, + "grad_norm": 0.31325674057006836, + "kl": 0.08090350031852722, + "learning_rate": 1.99989770512979e-05, + "loss": 0.0032, + "reward": 1.2827500104904175, + "reward_std": 1.164240837097168, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.2827500104904175, + "step": 258 + }, + { + "completion_length": 90.75, + "epoch": 0.03465810250234176, + "grad_norm": 0.40309247374534607, + "kl": 0.1473146677017212, + "learning_rate": 1.999891411619908e-05, + "loss": 0.0059, + "reward": 3.0, + "reward_std": 1.0, + "rewards/correctness_reward_func": 1.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.5, + "rewards/xmlcount_reward_func": 0.5, + "step": 259 + }, + { + "completion_length": 194.75, + "epoch": 0.034791917569918375, + "grad_norm": 0.30801522731781006, + "kl": 0.05805174261331558, + "learning_rate": 1.9998849302578597e-05, + "loss": 0.0023, + "reward": 1.8217499256134033, + "reward_std": 1.0732003450393677, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.125, + "rewards/xmlcount_reward_func": 0.1967500001192093, + "step": 260 + }, + { + "completion_length": 84.25, + "epoch": 0.03492573263749498, + "grad_norm": 0.7973276376724243, + "kl": 0.12621484696865082, + "learning_rate": 1.9998782610448625e-05, + "loss": 0.005, + "reward": 2.209249973297119, + "reward_std": 0.821670413017273, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.25, + "rewards/xmlcount_reward_func": 0.45925000309944153, + "step": 261 + }, + { + "completion_length": 182.25, + "epoch": 0.03505954770507159, + "grad_norm": 0.40601614117622375, + "kl": 0.1828838288784027, + "learning_rate": 1.9998714039821703e-05, + "loss": 0.0073, + "reward": 1.312999963760376, + "reward_std": 1.4628314971923828, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.125, + "rewards/xmlcount_reward_func": 0.18799999356269836, + "step": 262 + }, + { + "completion_length": 125.25, + "epoch": 0.0351933627726482, + "grad_norm": 0.4140985608100891, + "kl": 0.0889941155910492, + "learning_rate": 1.9998643590710707e-05, + "loss": 0.0036, + "reward": 3.125, + "reward_std": 0.4787135720252991, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.25, + "rewards/xmlcount_reward_func": 0.375, + "step": 263 + }, + { + "completion_length": 295.75, + "epoch": 0.03532717784022481, + "grad_norm": 0.24423445761203766, + "kl": 0.05636545270681381, + "learning_rate": 1.9998571263128873e-05, + "loss": 0.0023, + "reward": 0.24674999713897705, + "reward_std": 0.20656456053256989, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.12825000286102295, + "step": 264 + }, + { + "completion_length": 177.75, + "epoch": 0.03546099290780142, + "grad_norm": 0.4541739821434021, + "kl": 0.09995388984680176, + "learning_rate": 1.999849705708979e-05, + "loss": 0.004, + "reward": 0.7145000100135803, + "reward_std": 0.2543770372867584, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.21450001001358032, + "step": 265 + }, + { + "completion_length": 250.0, + "epoch": 0.03559480797537803, + "grad_norm": 0.2651154398918152, + "kl": 0.0614202618598938, + "learning_rate": 1.99984209726074e-05, + "loss": 0.0025, + "reward": 0.718500018119812, + "reward_std": 0.25749239325523376, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.21850000321865082, + "step": 266 + }, + { + "completion_length": 151.75, + "epoch": 0.035728623042954634, + "grad_norm": 0.34357938170433044, + "kl": 0.11051115393638611, + "learning_rate": 1.9998343009695995e-05, + "loss": 0.0044, + "reward": 1.125, + "reward_std": 0.9464846849441528, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.125, + "step": 267 + }, + { + "completion_length": 178.75, + "epoch": 0.03586243811053125, + "grad_norm": 0.6356491446495056, + "kl": 0.16408377885818481, + "learning_rate": 1.9998263168370228e-05, + "loss": 0.0066, + "reward": 0.4662500023841858, + "reward_std": 0.26134318113327026, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.03375000134110451, + "step": 268 + }, + { + "completion_length": 114.75, + "epoch": 0.035996253178107854, + "grad_norm": 0.5377089977264404, + "kl": 0.16050302982330322, + "learning_rate": 1.9998181448645087e-05, + "loss": 0.0064, + "reward": 2.5269999504089355, + "reward_std": 1.4152441024780273, + "rewards/correctness_reward_func": 1.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.25, + "rewards/xmlcount_reward_func": 0.2770000100135803, + "step": 269 + }, + { + "completion_length": 123.0, + "epoch": 0.03613006824568447, + "grad_norm": 0.3413686752319336, + "kl": 0.0774858370423317, + "learning_rate": 1.999809785053594e-05, + "loss": 0.0031, + "reward": 2.625, + "reward_std": 0.8539125919342041, + "rewards/correctness_reward_func": 1.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.25, + "rewards/xmlcount_reward_func": 0.375, + "step": 270 + }, + { + "completion_length": 78.0, + "epoch": 0.036263883313261074, + "grad_norm": 0.6230402588844299, + "kl": 0.14687515795230865, + "learning_rate": 1.999801237405848e-05, + "loss": 0.0059, + "reward": 3.0, + "reward_std": 1.0, + "rewards/correctness_reward_func": 1.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.5, + "rewards/xmlcount_reward_func": 0.5, + "step": 271 + }, + { + "completion_length": 240.5, + "epoch": 0.03639769838083768, + "grad_norm": 0.36297154426574707, + "kl": 0.09850015491247177, + "learning_rate": 1.9997925019228775e-05, + "loss": 0.0039, + "reward": 0.5189999938011169, + "reward_std": 0.9096706509590149, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.125, + "rewards/xmlcount_reward_func": 0.018999993801116943, + "step": 272 + }, + { + "completion_length": 167.75, + "epoch": 0.03653151344841429, + "grad_norm": 0.2768608629703522, + "kl": 0.06528370082378387, + "learning_rate": 1.999783578606323e-05, + "loss": 0.0026, + "reward": 1.0750000476837158, + "reward_std": 1.1177325248718262, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.07500000298023224, + "step": 273 + }, + { + "completion_length": 205.25, + "epoch": 0.0366653285159909, + "grad_norm": 0.24607378244400024, + "kl": 0.0492008738219738, + "learning_rate": 1.9997744674578615e-05, + "loss": 0.002, + "reward": 1.375, + "reward_std": 1.25, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.125, + "rewards/xmlcount_reward_func": 0.375, + "step": 274 + }, + { + "completion_length": 123.75, + "epoch": 0.036799143583567506, + "grad_norm": 0.3026469647884369, + "kl": 0.11610330641269684, + "learning_rate": 1.9997651684792042e-05, + "loss": 0.0046, + "reward": 2.647249937057495, + "reward_std": 0.850723385810852, + "rewards/correctness_reward_func": 1.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.25, + "rewards/xmlcount_reward_func": 0.3972499966621399, + "step": 275 + }, + { + "completion_length": 191.75, + "epoch": 0.03693295865114412, + "grad_norm": 0.2089061737060547, + "kl": 0.05618685111403465, + "learning_rate": 1.9997556816720985e-05, + "loss": 0.0022, + "reward": 1.6702499389648438, + "reward_std": 1.4511858224868774, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.125, + "rewards/xmlcount_reward_func": 0.045249998569488525, + "step": 276 + }, + { + "completion_length": 207.25, + "epoch": 0.037066773718720726, + "grad_norm": 0.25519034266471863, + "kl": 0.030937891453504562, + "learning_rate": 1.9997460070383264e-05, + "loss": 0.0012, + "reward": 0.9837499856948853, + "reward_std": 0.9404298067092896, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.01625000685453415, + "step": 277 + }, + { + "completion_length": 110.5, + "epoch": 0.03720058878629734, + "grad_norm": 0.2822895050048828, + "kl": 0.06590020656585693, + "learning_rate": 1.9997361445797058e-05, + "loss": 0.0026, + "reward": 2.0512499809265137, + "reward_std": 1.4043407440185547, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.125, + "rewards/xmlcount_reward_func": 0.42625001072883606, + "step": 278 + }, + { + "completion_length": 195.0, + "epoch": 0.037334403853873946, + "grad_norm": 0.3637947738170624, + "kl": 0.08848227560520172, + "learning_rate": 1.9997260942980895e-05, + "loss": 0.0035, + "reward": 1.3247499465942383, + "reward_std": 0.9170319437980652, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.125, + "rewards/xmlcount_reward_func": -0.17524999380111694, + "step": 279 + }, + { + "completion_length": 155.75, + "epoch": 0.03746821892145055, + "grad_norm": 0.3750488758087158, + "kl": 0.0733165591955185, + "learning_rate": 1.9997158561953655e-05, + "loss": 0.0029, + "reward": 2.0625, + "reward_std": 1.328768253326416, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.25, + "rewards/xmlcount_reward_func": 0.3125, + "step": 280 + }, + { + "completion_length": 173.75, + "epoch": 0.037602033989027166, + "grad_norm": 0.2433202862739563, + "kl": 0.07221105694770813, + "learning_rate": 1.9997054302734576e-05, + "loss": 0.0029, + "reward": 3.1524999141693115, + "reward_std": 0.4316924810409546, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.25, + "rewards/xmlcount_reward_func": 0.4025000035762787, + "step": 281 + }, + { + "completion_length": 173.25, + "epoch": 0.03773584905660377, + "grad_norm": 0.2556951940059662, + "kl": 0.043353110551834106, + "learning_rate": 1.9996948165343243e-05, + "loss": 0.0017, + "reward": 0.7929999828338623, + "reward_std": 0.29227155447006226, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.2929999828338623, + "step": 282 + }, + { + "completion_length": 127.0, + "epoch": 0.037869664124180386, + "grad_norm": 0.2885083556175232, + "kl": 0.06136604771018028, + "learning_rate": 1.9996840149799594e-05, + "loss": 0.0025, + "reward": 1.2899999618530273, + "reward_std": 0.9839661717414856, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.2900000214576721, + "step": 283 + }, + { + "completion_length": 119.75, + "epoch": 0.03800347919175699, + "grad_norm": 0.4060731828212738, + "kl": 0.0710485652089119, + "learning_rate": 1.9996730256123925e-05, + "loss": 0.0028, + "reward": 1.2477500438690186, + "reward_std": 0.7362886667251587, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.24774999916553497, + "step": 284 + }, + { + "completion_length": 117.25, + "epoch": 0.0381372942593336, + "grad_norm": 0.32145437598228455, + "kl": 0.06920292973518372, + "learning_rate": 1.9996618484336885e-05, + "loss": 0.0028, + "reward": 1.3524999618530273, + "reward_std": 0.9444156885147095, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.3525000214576721, + "step": 285 + }, + { + "completion_length": 145.0, + "epoch": 0.03827110932691021, + "grad_norm": 0.29436415433883667, + "kl": 0.06218549609184265, + "learning_rate": 1.9996504834459467e-05, + "loss": 0.0025, + "reward": 1.503749966621399, + "reward_std": 1.1761995553970337, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.125, + "rewards/xmlcount_reward_func": 0.1287499964237213, + "step": 286 + }, + { + "completion_length": 87.0, + "epoch": 0.03840492439448682, + "grad_norm": 0.520235538482666, + "kl": 0.13112740218639374, + "learning_rate": 1.9996389306513028e-05, + "loss": 0.0052, + "reward": 2.296999931335449, + "reward_std": 0.8679065704345703, + "rewards/correctness_reward_func": 1.5, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.125, + "rewards/xmlcount_reward_func": 0.296999990940094, + "step": 287 + }, + { + "completion_length": 200.25, + "epoch": 0.03853873946206343, + "grad_norm": 0.21292062103748322, + "kl": 0.03701108694076538, + "learning_rate": 1.9996271900519267e-05, + "loss": 0.0015, + "reward": 0.9787499904632568, + "reward_std": 0.8390643000602722, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.02124999463558197, + "step": 288 + }, + { + "completion_length": 197.0, + "epoch": 0.03867255452964004, + "grad_norm": 0.3041883707046509, + "kl": 0.04169435426592827, + "learning_rate": 1.9996152616500244e-05, + "loss": 0.0017, + "reward": 1.4789999723434448, + "reward_std": 1.2745074033737183, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.021000005304813385, + "step": 289 + }, + { + "completion_length": 116.5, + "epoch": 0.038806369597216644, + "grad_norm": 0.5523695349693298, + "kl": 0.04961012303829193, + "learning_rate": 1.999603145447837e-05, + "loss": 0.002, + "reward": 2.204249858856201, + "reward_std": 0.949548065662384, + "rewards/correctness_reward_func": 1.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.2042500078678131, + "step": 290 + }, + { + "completion_length": 162.25, + "epoch": 0.03894018466479326, + "grad_norm": 0.3126446008682251, + "kl": 0.08022642135620117, + "learning_rate": 1.999590841447641e-05, + "loss": 0.0032, + "reward": 1.3469998836517334, + "reward_std": 1.3204811811447144, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.02800000086426735, + "step": 291 + }, + { + "completion_length": 131.75, + "epoch": 0.039073999732369864, + "grad_norm": 0.20107072591781616, + "kl": 0.0379381999373436, + "learning_rate": 1.9995783496517476e-05, + "loss": 0.0015, + "reward": 0.2239999920129776, + "reward_std": 0.13086380064487457, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.2239999920129776, + "step": 292 + }, + { + "completion_length": 189.5, + "epoch": 0.03920781479994647, + "grad_norm": 0.20484226942062378, + "kl": 0.03132232278585434, + "learning_rate": 1.999565670062504e-05, + "loss": 0.0013, + "reward": 0.7402499914169312, + "reward_std": 0.5397964715957642, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.125, + "rewards/xmlcount_reward_func": 0.11524999141693115, + "step": 293 + }, + { + "completion_length": 205.75, + "epoch": 0.039341629867523084, + "grad_norm": 0.24561399221420288, + "kl": 0.0185256190598011, + "learning_rate": 1.9995528026822916e-05, + "loss": 0.0007, + "reward": 1.4252500534057617, + "reward_std": 1.6236603260040283, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.07475000619888306, + "step": 294 + }, + { + "completion_length": 201.25, + "epoch": 0.03947544493509969, + "grad_norm": 0.3286190629005432, + "kl": 0.031586844474077225, + "learning_rate": 1.999539747513529e-05, + "loss": 0.0013, + "reward": 2.189500093460083, + "reward_std": 0.9647402763366699, + "rewards/correctness_reward_func": 1.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.18950000405311584, + "step": 295 + }, + { + "completion_length": 121.0, + "epoch": 0.039609260002676304, + "grad_norm": 0.3337399661540985, + "kl": 0.03786204382777214, + "learning_rate": 1.999526504558668e-05, + "loss": 0.0015, + "reward": 2.154250144958496, + "reward_std": 0.9408127069473267, + "rewards/correctness_reward_func": 1.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.15424999594688416, + "step": 296 + }, + { + "completion_length": 106.5, + "epoch": 0.03974307507025291, + "grad_norm": 0.37861064076423645, + "kl": 0.062134772539138794, + "learning_rate": 1.9995130738201966e-05, + "loss": 0.0025, + "reward": 1.1582499742507935, + "reward_std": 0.9733763933181763, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.15825000405311584, + "step": 297 + }, + { + "completion_length": 243.75, + "epoch": 0.03987689013782952, + "grad_norm": 0.16624334454536438, + "kl": 0.03450315073132515, + "learning_rate": 1.9994994553006386e-05, + "loss": 0.0014, + "reward": 0.6399999856948853, + "reward_std": 1.6276360750198364, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.10999999940395355, + "step": 298 + }, + { + "completion_length": 93.25, + "epoch": 0.04001070520540613, + "grad_norm": 0.4910159707069397, + "kl": 0.04987555742263794, + "learning_rate": 1.999485649002552e-05, + "loss": 0.002, + "reward": 0.9787499904632568, + "reward_std": 1.2580289840698242, + "rewards/correctness_reward_func": 0.5, + "rewards/int_reward_func": 0.375, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.10375000536441803, + "step": 299 + }, + { + "completion_length": 247.5, + "epoch": 0.04014452027298274, + "grad_norm": 0.21725068986415863, + "kl": 0.011182424612343311, + "learning_rate": 1.9994716549285312e-05, + "loss": 0.0004, + "reward": 1.8627500534057617, + "reward_std": 0.7289958000183105, + "rewards/correctness_reward_func": 1.5, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.13725000619888306, + "step": 300 + } + ], + "logging_steps": 1, + "max_steps": 7473, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +}