Upload training_log.json with huggingface_hub

training_log.json

@@ -2,1441 +2,1351 @@
   {
     "update": 5,
     "global_step": 10240,
-    "num_episodes": 5,
-    "mean_reward": -2.121724510192871,
-    "mean_length": 337.8,
-    "loss": 2.8168609142303467,
-    "sps": 228.7546335682378
+    "num_episodes": 12,
+    "mean_reward": -1.8039220174153645,
+    "mean_length": 214.16666666666666,
+    "loss": 1.5202983617782593,
+    "sps": 1350.1393046305259
   },
   {
     "update": 10,
     "global_step": 20480,
-    "num_episodes": 18,
-    "mean_reward": -1.9787551561991374,
-    "mean_length": 282.94444444444446,
-    "loss": 0.21835778653621674,
-    "sps": 1443.696434210723
+    "num_episodes": 12,
+    "mean_reward": -1.8039220174153645,
+    "mean_length": 214.16666666666666,
+    "loss": 0.6584598422050476,
+    "sps": 2047.2883333230054
   },
   {
     "update": 15,
     "global_step": 30720,
-    "num_episodes": 18,
-    "mean_reward": -1.9787551561991374,
-    "mean_length": 282.94444444444446,
-    "loss": 0.2217431664466858,
-    "sps": 2444.104440024094
+    "num_episodes": 12,
+    "mean_reward": -1.8039220174153645,
+    "mean_length": 214.16666666666666,
+    "loss": 0.6658369898796082,
+    "sps": 2055.41935946998
   },
   {
     "update": 20,
     "global_step": 40960,
-    "num_episodes": 22,
-    "mean_reward": 0.6390173651955344,
-    "mean_length": 1597.090909090909,
-    "loss": 0.5035408735275269,
-    "sps": 769.1328942825502
+    "num_episodes": 15,
+    "mean_reward": 2.1853319803873696,
+    "mean_length": 1510.9333333333334,
+    "loss": 1.3961129188537598,
+    "sps": 1412.8079652055148
   },
   {
     "update": 25,
     "global_step": 51200,
-    "num_episodes": 27,
-    "mean_reward": 0.33698498761212387,
-    "mean_length": 1341.111111111111,
-    "loss": 0.16991910338401794,
-    "sps": 2449.665853907546
+    "num_episodes": 22,
+    "mean_reward": 2.625577276403254,
+    "mean_length": 1983.409090909091,
+    "loss": 1.2982873916625977,
+    "sps": 470.35650512927947
   },
   {
     "update": 30,
     "global_step": 61440,
-    "num_episodes": 29,
-    "mean_reward": 0.5858152652608937,
-    "mean_length": 1594.9310344827586,
-    "loss": 0.6312248110771179,
-    "sps": 2363.5499282399865
+    "num_episodes": 22,
+    "mean_reward": 2.625577276403254,
+    "mean_length": 1983.409090909091,
+    "loss": 1.0302618741989136,
+    "sps": 2216.3140512542655
   },
   {
     "update": 35,
     "global_step": 71680,
-    "num_episodes": 29,
-    "mean_reward": 0.5858152652608937,
-    "mean_length": 1594.9310344827586,
-    "loss": 0.4890201985836029,
-    "sps": 2440.672480568945
+    "num_episodes": 22,
+    "mean_reward": 2.625577276403254,
+    "mean_length": 1983.409090909091,
+    "loss": 0.9787145853042603,
+    "sps": 2070.449469801819
   },
   {
     "update": 40,
     "global_step": 81920,
-    "num_episodes": 31,
-    "mean_reward": 1.2613418640628937,
-    "mean_length": 1822.0,
-    "loss": 0.42574575543403625,
-    "sps": 940.0717318496648
+    "num_episodes": 27,
+    "mean_reward": 3.300225363837348,
+    "mean_length": 2365.4074074074074,
+    "loss": 0.8183227777481079,
+    "sps": 755.8004689166112
   },
   {
     "update": 45,
     "global_step": 92160,
-    "num_episodes": 55,
-    "mean_reward": 2.8606436122547496,
-    "mean_length": 1469.2727272727273,
-    "loss": 0.5428293943405151,
-    "sps": 1479.6348998060785
+    "num_episodes": 33,
+    "mean_reward": 3.7748724330555308,
+    "mean_length": 2564.090909090909,
+    "loss": 0.3192096948623657,
+    "sps": 1984.8729329667028
   },
   {
     "update": 50,
     "global_step": 102400,
-    "num_episodes": 58,
-    "mean_reward": 3.2600761939739358,
-    "mean_length": 1568.051724137931,
-    "loss": 0.6221064329147339,
-    "sps": 2412.4523346060355
+    "num_episodes": 33,
+    "mean_reward": 3.7748724330555308,
+    "mean_length": 2564.090909090909,
+    "loss": 0.8334144949913025,
+    "sps": 2085.15978587018
   },
   {
     "update": 55,
     "global_step": 112640,
-    "num_episodes": 58,
-    "mean_reward": 3.2600761939739358,
-    "mean_length": 1568.051724137931,
-    "loss": 0.794475257396698,
-    "sps": 2405.0877911326343
+    "num_episodes": 33,
+    "mean_reward": 3.7748724330555308,
+    "mean_length": 2564.090909090909,
+    "loss": 0.5191329121589661,
+    "sps": 2075.918435842757
   },
   {
     "update": 60,
     "global_step": 122880,
-    "num_episodes": 58,
-    "mean_reward": 3.2600761939739358,
-    "mean_length": 1568.051724137931,
-    "loss": 1.0936992168426514,
-    "sps": 2358.874803208204
+    "num_episodes": 35,
+    "mean_reward": 5.517268398829869,
+    "mean_length": 2989.0,
+    "loss": 0.6924473643302917,
+    "sps": 2020.4075872880348
   },
   {
     "update": 65,
     "global_step": 133120,
-    "num_episodes": 62,
-    "mean_reward": 7.5118274227265385,
-    "mean_length": 1954.4516129032259,
-    "loss": 1.877166986465454,
-    "sps": 2402.6606220225244
+    "num_episodes": 40,
+    "mean_reward": 5.594199681282044,
+    "mean_length": 3128.125,
+    "loss": 1.9699269533157349,
+    "sps": 429.94535608180354
   },
   {
     "update": 70,
     "global_step": 143360,
-    "num_episodes": 63,
-    "mean_reward": 9.12682440924266,
-    "mean_length": 2082.15873015873,
-    "loss": 0.55415278673172,
-    "sps": 2417.4656020731263
+    "num_episodes": 41,
+    "mean_reward": 5.430626520296422,
+    "mean_length": 3061.170731707317,
+    "loss": 2.169956684112549,
+    "sps": 2246.502749698459
   },
   {
     "update": 75,
     "global_step": 153600,
-    "num_episodes": 63,
-    "mean_reward": 9.12682440924266,
-    "mean_length": 2082.15873015873,
-    "loss": 0.6123632788658142,
-    "sps": 2415.520694055524
+    "num_episodes": 41,
+    "mean_reward": 5.430626520296422,
+    "mean_length": 3061.170731707317,
+    "loss": 2.5021121501922607,
+    "sps": 2240.800087024051
   },
   {
     "update": 80,
     "global_step": 163840,
-    "num_episodes": 63,
-    "mean_reward": 9.12682440924266,
-    "mean_length": 2082.15873015873,
-    "loss": 0.3689587116241455,
-    "sps": 2464.6240808517623
+    "num_episodes": 46,
+    "mean_reward": 10.588201502095098,
+    "mean_length": 3177.4565217391305,
+    "loss": 2.6493680477142334,
+    "sps": 436.6488855176695
   },
   {
     "update": 85,
     "global_step": 174080,
-    "num_episodes": 70,
-    "mean_reward": 9.247220189230783,
-    "mean_length": 2310.557142857143,
-    "loss": 0.3228328824043274,
-    "sps": 2445.2962822104055
+    "num_episodes": 52,
+    "mean_reward": 13.96443979556744,
+    "mean_length": 3211.1923076923076,
+    "loss": 2.5603184700012207,
+    "sps": 2163.643359340071
   },
   {
     "update": 90,
     "global_step": 184320,
-    "num_episodes": 76,
-    "mean_reward": 9.205821903128372,
-    "mean_length": 2268.3026315789475,
-    "loss": 0.6627960205078125,
-    "sps": 453.4183326926133
+    "num_episodes": 52,
+    "mean_reward": 13.96443979556744,
+    "mean_length": 3211.1923076923076,
+    "loss": 2.336817502975464,
+    "sps": 2163.4161260603714
   },
   {
     "update": 95,
     "global_step": 194560,
-    "num_episodes": 80,
-    "mean_reward": 9.297839653491973,
-    "mean_length": 2165.9375,
-    "loss": 1.1097347736358643,
-    "sps": 2424.5065546850346
+    "num_episodes": 52,
+    "mean_reward": 13.96443979556744,
+    "mean_length": 3211.1923076923076,
+    "loss": 3.234126091003418,
+    "sps": 2186.3643111360093
   },
   {
     "update": 100,
     "global_step": 204800,
-    "num_episodes": 80,
-    "mean_reward": 9.297839653491973,
-    "mean_length": 2165.9375,
-    "loss": 0.38644012808799744,
-    "sps": 2386.6710469869367
+    "num_episodes": 55,
+    "mean_reward": 17.41634755568071,
+    "mean_length": 3220.3454545454547,
+    "loss": 5.442279815673828,
+    "sps": 1960.8741209669945
   },
   {
     "update": 105,
     "global_step": 215040,
-    "num_episodes": 87,
-    "mean_reward": 11.394574307847297,
-    "mean_length": 2339.057471264368,
-    "loss": 0.7209265828132629,
-    "sps": 2442.8262241524835
+    "num_episodes": 62,
+    "mean_reward": 27.910769308767012,
+    "mean_length": 3345.3387096774195,
+    "loss": 2.81072735786438,
+    "sps": 1303.1946553744522
   },
   {
     "update": 110,
     "global_step": 225280,
-    "num_episodes": 87,
-    "mean_reward": 11.394574307847297,
-    "mean_length": 2339.057471264368,
-    "loss": 2.1218621730804443,
-    "sps": 2449.9984290228053
+    "num_episodes": 66,
+    "mean_reward": 26.517480315584123,
+    "mean_length": 3151.8939393939395,
+    "loss": 0.830104649066925,
+    "sps": 1994.5176102850091
   },
   {
     "update": 115,
     "global_step": 235520,
-    "num_episodes": 90,
-    "mean_reward": 12.198311275906033,
-    "mean_length": 2375.788888888889,
-    "loss": 0.5268573760986328,
-    "sps": 2438.632604283566
+    "num_episodes": 66,
+    "mean_reward": 26.517480315584123,
+    "mean_length": 3151.8939393939395,
+    "loss": 0.8076149225234985,
+    "sps": 2020.1985152956534
   },
   {
     "update": 120,
     "global_step": 245760,
-    "num_episodes": 90,
-    "mean_reward": 12.198311275906033,
-    "mean_length": 2375.788888888889,
-    "loss": 2.0001754760742188,
-    "sps": 2413.1198849334774
+    "num_episodes": 69,
+    "mean_reward": 35.54813836968464,
+    "mean_length": 3449.6376811594205,
+    "loss": 1.7053556442260742,
+    "sps": 1392.6768298802306
   },
   {
     "update": 125,
     "global_step": 256000,
-    "num_episodes": 105,
-    "mean_reward": 12.691367435455323,
-    "mean_length": 2446.14,
-    "loss": 0.6336287260055542,
-    "sps": 619.7035033065293
+    "num_episodes": 72,
+    "mean_reward": 34.35707297590044,
+    "mean_length": 3315.0,
+    "loss": 0.5338165163993835,
+    "sps": 2218.857641762577
   },
   {
     "update": 130,
     "global_step": 266240,
-    "num_episodes": 105,
-    "mean_reward": 12.691367435455323,
-    "mean_length": 2446.14,
-    "loss": 0.40229156613349915,
-    "sps": 2411.1751921586524
+    "num_episodes": 73,
+    "mean_reward": 37.437843152921495,
+    "mean_length": 3406.5753424657532,
+    "loss": 0.31061413884162903,
+    "sps": 2123.373275302262
   },
   {
     "update": 135,
     "global_step": 276480,
-    "num_episodes": 109,
-    "mean_reward": 13.886511907577514,
-    "mean_length": 2526.42,
-    "loss": 0.45471617579460144,
-    "sps": 2450.150074118317
+    "num_episodes": 73,
+    "mean_reward": 37.437843152921495,
+    "mean_length": 3406.5753424657532,
+    "loss": 0.370238333940506,
+    "sps": 2270.1968695887504
   },
   {
     "update": 140,
     "global_step": 286720,
-    "num_episodes": 109,
-    "mean_reward": 13.886511907577514,
-    "mean_length": 2526.42,
-    "loss": 0.4323250949382782,
-    "sps": 2378.2401254524298
+    "num_episodes": 76,
+    "mean_reward": 42.764281247791494,
+    "mean_length": 3535.8289473684213,
+    "loss": 0.4431289732456207,
+    "sps": 2235.4805453156696
   },
   {
     "update": 145,
     "global_step": 296960,
-    "num_episodes": 121,
-    "mean_reward": 14.005602750778198,
-    "mean_length": 2623.1,
-    "loss": 0.5164788365364075,
-    "sps": 1792.6912302713768
+    "num_episodes": 78,
+    "mean_reward": 45.29765256245931,
+    "mean_length": 3576.3076923076924,
+    "loss": 0.11675499379634857,
+    "sps": 2248.214328955258
   },
   {
     "update": 150,
     "global_step": 307200,
-    "num_episodes": 121,
-    "mean_reward": 14.005602750778198,
-    "mean_length": 2623.1,
-    "loss": 0.8702018857002258,
-    "sps": 2425.950637318183
+    "num_episodes": 79,
+    "mean_reward": 48.32240367841117,
+    "mean_length": 3657.6202531645567,
+    "loss": 0.3231047987937927,
+    "sps": 2273.3126392381514
   },
   {
     "update": 155,
     "global_step": 317440,
-    "num_episodes": 129,
-    "mean_reward": 14.87307053565979,
-    "mean_length": 2527.32,
-    "loss": 0.1953483670949936,
-    "sps": 2394.950934358607
+    "num_episodes": 79,
+    "mean_reward": 48.32240367841117,
+    "mean_length": 3657.6202531645567,
+    "loss": 0.24974392354488373,
+    "sps": 2276.2572089535233
   },
   {
     "update": 160,
     "global_step": 327680,
-    "num_episodes": 129,
-    "mean_reward": 14.87307053565979,
-    "mean_length": 2527.32,
-    "loss": 0.19992592930793762,
-    "sps": 2424.040624859784
+    "num_episodes": 90,
+    "mean_reward": 48.88773588604397,
+    "mean_length": 3444.8333333333335,
+    "loss": 1.1440778970718384,
+    "sps": 469.25381312597585
   },
   {
     "update": 165,
     "global_step": 337920,
-    "num_episodes": 139,
-    "mean_reward": 16.099841012954712,
-    "mean_length": 2525.93,
-    "loss": 0.4464109241962433,
-    "sps": 1319.6508401584945
+    "num_episodes": 99,
+    "mean_reward": 50.101470494511155,
+    "mean_length": 3348.060606060606,
+    "loss": 0.3425019383430481,
+    "sps": 1903.5186981694392
   },
   {
     "update": 170,
     "global_step": 348160,
-    "num_episodes": 139,
-    "mean_reward": 16.099841012954712,
-    "mean_length": 2525.93,
-    "loss": 0.10373280942440033,
-    "sps": 2348.0679386786046
+    "num_episodes": 99,
+    "mean_reward": 50.101470494511155,
+    "mean_length": 3348.060606060606,
+    "loss": 0.18799301981925964,
+    "sps": 2130.3254740506277
   },
   {
     "update": 175,
     "global_step": 358400,
-    "num_episodes": 142,
-    "mean_reward": 17.19686735153198,
-    "mean_length": 2624.69,
-    "loss": 0.8095718026161194,
-    "sps": 2386.2480483453646
+    "num_episodes": 99,
+    "mean_reward": 50.101470494511155,
+    "mean_length": 3348.060606060606,
+    "loss": 0.11751141399145126,
+    "sps": 2125.2682193036194
   },
   {
     "update": 180,
     "global_step": 368640,
-    "num_episodes": 142,
-    "mean_reward": 17.19686735153198,
-    "mean_length": 2624.69,
-    "loss": 0.009909076616168022,
-    "sps": 2301.590679746089
+    "num_episodes": 101,
+    "mean_reward": 52.442985696792604,
+    "mean_length": 3414.58,
+    "loss": 0.2516184449195862,
+    "sps": 1703.135913858338
   },
   {
     "update": 185,
     "global_step": 378880,
-    "num_episodes": 148,
-    "mean_reward": 19.868873529434204,
-    "mean_length": 2907.66,
-    "loss": 1.7244997024536133,
-    "sps": 2280.4958488818847
+    "num_episodes": 114,
+    "mean_reward": 61.27311011314392,
+    "mean_length": 3506.98,
+    "loss": 0.11899760365486145,
+    "sps": 2246.894695169401
   },
   {
     "update": 190,
     "global_step": 389120,
-    "num_episodes": 148,
-    "mean_reward": 19.868873529434204,
-    "mean_length": 2907.66,
-    "loss": 4.516170978546143,
-    "sps": 2419.553348740851
+    "num_episodes": 114,
+    "mean_reward": 61.27311011314392,
+    "mean_length": 3506.98,
+    "loss": 0.30889129638671875,
+    "sps": 2289.0664114120555
   },
   {
     "update": 195,
     "global_step": 399360,
-    "num_episodes": 152,
-    "mean_reward": 20.717632093429565,
-    "mean_length": 3008.33,
-    "loss": 3.401921272277832,
-    "sps": 575.3298806579921
+    "num_episodes": 114,
+    "mean_reward": 61.27311011314392,
+    "mean_length": 3506.98,
+    "loss": -0.06268183887004852,
+    "sps": 2277.013858345863
   },
   {
     "update": 200,
     "global_step": 409600,
-    "num_episodes": 153,
-    "mean_reward": 20.717632093429565,
-    "mean_length": 3008.33,
-    "loss": 0.8132290244102478,
-    "sps": 2450.768029758656
+    "num_episodes": 114,
+    "mean_reward": 61.27311011314392,
+    "mean_length": 3506.98,
+    "loss": 0.5904355645179749,
+    "sps": 2173.4262065580283
   },
   {
     "update": 205,
     "global_step": 419840,
-    "num_episodes": 162,
-    "mean_reward": 19.608867626190186,
-    "mean_length": 2909.79,
-    "loss": 1.867810845375061,
-    "sps": 621.2303877555788
+    "num_episodes": 121,
+    "mean_reward": 72.71755365371705,
+    "mean_length": 3714.22,
+    "loss": 1.7067689895629883,
+    "sps": 308.3386383776914
   },
   {
     "update": 210,
     "global_step": 430080,
-    "num_episodes": 167,
-    "mean_reward": 19.028235759735107,
-    "mean_length": 2619.36,
-    "loss": 0.8309984803199768,
-    "sps": 2358.6319149765754
+    "num_episodes": 123,
+    "mean_reward": 72.60844767570495,
+    "mean_length": 3634.11,
+    "loss": 0.22778448462486267,
+    "sps": 2270.040885359684
   },
   {
     "update": 215,
     "global_step": 440320,
-    "num_episodes": 169,
-    "mean_reward": 19.63686834335327,
-    "mean_length": 2715.53,
-    "loss": 0.17013275623321533,
-    "sps": 1381.2913503327266
+    "num_episodes": 123,
+    "mean_reward": 72.60844767570495,
+    "mean_length": 3634.11,
+    "loss": 0.4201895296573639,
+    "sps": 2286.5998087120915
   },
   {
     "update": 220,
     "global_step": 450560,
-    "num_episodes": 170,
-    "mean_reward": 19.778069705963134,
-    "mean_length": 2619.36,
-    "loss": -0.013407116755843163,
-    "sps": 2225.527904161147
+    "num_episodes": 123,
+    "mean_reward": 72.60844767570495,
+    "mean_length": 3634.11,
+    "loss": 0.08545871078968048,
+    "sps": 2301.79050438536
   },
   {
     "update": 225,
     "global_step": 460800,
-    "num_episodes": 172,
-    "mean_reward": 21.13547016143799,
-    "mean_length": 2718.42,
-    "loss": 0.14364227652549744,
-    "sps": 2345.342614298781
+    "num_episodes": 129,
+    "mean_reward": 81.06719317436219,
+    "mean_length": 3731.63,
+    "loss": 0.4734661877155304,
+    "sps": 1893.4799410834491
   },
   {
     "update": 230,
     "global_step": 471040,
-    "num_episodes": 174,
-    "mean_reward": 22.236838788986205,
-    "mean_length": 2819.34,
-    "loss": 0.017919447273015976,
-    "sps": 2408.348085488877
+    "num_episodes": 132,
+    "mean_reward": 83.9696240234375,
+    "mean_length": 3736.94,
+    "loss": 2.656114101409912,
+    "sps": 1747.8656326711334
   },
   {
     "update": 235,
     "global_step": 481280,
-    "num_episodes": 174,
-    "mean_reward": 22.236838788986205,
-    "mean_length": 2819.34,
-    "loss": 2.6431455612182617,
-    "sps": 2403.528538830992
+    "num_episodes": 132,
+    "mean_reward": 83.9696240234375,
+    "mean_length": 3736.94,
+    "loss": 0.3720909357070923,
+    "sps": 2290.767122681247
   },
   {
     "update": 240,
     "global_step": 491520,
-    "num_episodes": 178,
-    "mean_reward": 21.810234026908873,
-    "mean_length": 2928.26,
-    "loss": 0.2556005120277405,
-    "sps": 2373.833411448431
+    "num_episodes": 133,
+    "mean_reward": 86.32599678993225,
+    "mean_length": 3836.51,
+    "loss": 0.28272855281829834,
+    "sps": 1660.2388311365764
   },
   {
     "update": 245,
     "global_step": 501760,
-    "num_episodes": 181,
-    "mean_reward": 23.416525321006773,
-    "mean_length": 3026.48,
-    "loss": 3.2893059253692627,
-    "sps": 2362.3097327902087
+    "num_episodes": 139,
+    "mean_reward": 90.87246092796326,
+    "mean_length": 3642.75,
+    "loss": 0.7918240427970886,
+    "sps": 226.8334199523192
   },
   {
     "update": 250,
     "global_step": 512000,
-    "num_episodes": 182,
-    "mean_reward": 23.666512427330016,
-    "mean_length": 3026.48,
-    "loss": 4.634244441986084,
-    "sps": 2410.981639631563
+    "num_episodes": 144,
+    "mean_reward": 93.14730869293213,
+    "mean_length": 3661.34,
+    "loss": 0.5329251885414124,
+    "sps": 1291.7356099331766
   },
   {
     "update": 255,
     "global_step": 522240,
-    "num_episodes": 182,
-    "mean_reward": 23.666512427330016,
-    "mean_length": 3026.48,
-    "loss": 1.1017451286315918,
-    "sps": 2373.168403411863
+    "num_episodes": 147,
+    "mean_reward": 90.92609573364258,
+    "mean_length": 3465.71,
+    "loss": 0.2208552062511444,
+    "sps": 2176.5233668919136
   },
   {
     "update": 260,
     "global_step": 532480,
-    "num_episodes": 188,
-    "mean_reward": 24.153521933555602,
-    "mean_length": 3032.56,
-    "loss": 0.058166228234767914,
-    "sps": 549.3090191152681
+    "num_episodes": 148,
+    "mean_reward": 94.03146264076233,
+    "mean_length": 3565.28,
+    "loss": 0.3058362305164337,
+    "sps": 1287.827730215577
   },
   {
     "update": 265,
     "global_step": 542720,
-    "num_episodes": 200,
-    "mean_reward": 24.202102789878847,
-    "mean_length": 2827.14,
-    "loss": 3.6524577140808105,
-    "sps": 2408.936350313052
+    "num_episodes": 153,
+    "mean_reward": 93.66601790428162,
+    "mean_length": 3469.44,
+    "loss": 0.2177593857049942,
+    "sps": 2185.2068742705324
   },
   {
     "update": 270,
     "global_step": 552960,
-    "num_episodes": 203,
-    "mean_reward": 25.780276198387146,
-    "mean_length": 2924.21,
-    "loss": -0.03986305743455887,
-    "sps": 2384.8038232666454
+    "num_episodes": 154,
+    "mean_reward": 96.52238609313964,
+    "mean_length": 3569.01,
+    "loss": 0.1883123517036438,
+    "sps": 2191.858927925312
   },
   {
     "update": 275,
     "global_step": 563200,
-    "num_episodes": 203,
-    "mean_reward": 25.780276198387146,
-    "mean_length": 2924.21,
-    "loss": 0.009281929582357407,
-    "sps": 2245.2843029907467
+    "num_episodes": 155,
+    "mean_reward": 99.62512998580932,
+    "mean_length": 3668.07,
+    "loss": 0.27221783995628357,
+    "sps": 2123.45043596574
   },
   {
     "update": 280,
     "global_step": 573440,
-    "num_episodes": 204,
-    "mean_reward": 26.63669138431549,
-    "mean_length": 3023.78,
-    "loss": 0.1620613932609558,
-    "sps": 2155.8537501850938
+    "num_episodes": 155,
+    "mean_reward": 99.62512998580932,
+    "mean_length": 3668.07,
+    "loss": 0.18106494843959808,
+    "sps": 2141.5921280557186
   },
   {
     "update": 285,
     "global_step": 583680,
-    "num_episodes": 208,
-    "mean_reward": 27.24311915874481,
-    "mean_length": 3124.7,
-    "loss": 0.7804455161094666,
-    "sps": 2336.540118617715
+    "num_episodes": 162,
+    "mean_reward": 100.8679942893982,
+    "mean_length": 3670.27,
+    "loss": 0.5695434212684631,
+    "sps": 500.88139423117855
   },
   {
     "update": 290,
     "global_step": 593920,
-    "num_episodes": 210,
-    "mean_reward": 27.990238661766053,
-    "mean_length": 3122.84,
-    "loss": -0.02732567861676216,
-    "sps": 2446.515068539969
+    "num_episodes": 162,
+    "mean_reward": 100.8679942893982,
+    "mean_length": 3670.27,
+    "loss": 0.07588323950767517,
+    "sps": 2141.039120248053
   },
   {
     "update": 295,
     "global_step": 604160,
-    "num_episodes": 210,
-    "mean_reward": 27.990238661766053,
-    "mean_length": 3122.84,
-    "loss": 0.026210829615592957,
-    "sps": 2401.0219632291446
+    "num_episodes": 164,
+    "mean_reward": 103.97303117752075,
+    "mean_length": 3770.47,
+    "loss": 0.3367934823036194,
+    "sps": 2232.451302693334
   },
   {
     "update": 300,
     "global_step": 614400,
-    "num_episodes": 212,
-    "mean_reward": 28.8441028547287,
-    "mean_length": 3221.9,
-    "loss": 0.018815483897924423,
-    "sps": 1787.985104867478
+    "num_episodes": 164,
+    "mean_reward": 103.97303117752075,
+    "mean_length": 3770.47,
+    "loss": 0.23896968364715576,
+    "sps": 2206.894347275839
   },
   {
     "update": 305,
     "global_step": 624640,
-    "num_episodes": 219,
-    "mean_reward": 29.601799569129945,
-    "mean_length": 3224.78,
-    "loss": -0.01376257836818695,
-    "sps": 2307.873344880848
+    "num_episodes": 168,
+    "mean_reward": 108.57997800827026,
+    "mean_length": 3866.64,
+    "loss": 0.8634886145591736,
+    "sps": 2068.617103764695
   },
   {
     "update": 310,
     "global_step": 634880,
-    "num_episodes": 223,
-    "mean_reward": 30.112299256324768,
-    "mean_length": 3222.81,
-    "loss": -0.13672460615634918,
-    "sps": 2376.26116166583
+    "num_episodes": 168,
+    "mean_reward": 108.57997800827026,
+    "mean_length": 3866.64,
+    "loss": 0.05964889004826546,
+    "sps": 2227.05463999608
   },
   {
     "update": 315,
     "global_step": 645120,
-    "num_episodes": 223,
-    "mean_reward": 30.112299256324768,
-    "mean_length": 3222.81,
-    "loss": 0.24129757285118103,
-    "sps": 2301.537029038706
+    "num_episodes": 170,
+    "mean_reward": 108.82997495651244,
+    "mean_length": 3866.64,
+    "loss": 0.3584131598472595,
+    "sps": 2093.8527505731936
   },
   {
     "update": 320,
     "global_step": 655360,
-    "num_episodes": 224,
-    "mean_reward": 30.965999827384948,
-    "mean_length": 3320.52,
-    "loss": 0.3078324496746063,
-    "sps": 2359.266120378419
+    "num_episodes": 170,
+    "mean_reward": 108.82997495651244,
+    "mean_length": 3866.64,
+    "loss": 0.11968313157558441,
+    "sps": 2108.9095828206277
   },
   {
     "update": 325,
     "global_step": 665600,
-    "num_episodes": 230,
-    "mean_reward": 31.090740485191347,
-    "mean_length": 3419.33,
-    "loss": 0.002869449555873871,
-    "sps": 2345.5360183144344
+    "num_episodes": 174,
+    "mean_reward": 112.43493017196656,
+    "mean_length": 3960.91,
+    "loss": 1.6604670286178589,
+    "sps": 2079.794147719775
   },
   {
     "update": 330,
     "global_step": 675840,
-    "num_episodes": 233,
-    "mean_reward": 31.204454226493837,
-    "mean_length": 3514.15,
-    "loss": 0.8130831718444824,
-    "sps": 2348.1224969342966
+    "num_episodes": 175,
+    "mean_reward": 115.29129805564881,
+    "mean_length": 4060.48,
+    "loss": 0.19095765054225922,
+    "sps": 2102.571665929904
   },
   {
     "update": 335,
     "global_step": 686080,
-    "num_episodes": 233,
-    "mean_reward": 31.204454226493837,
-    "mean_length": 3514.15,
-    "loss": -0.10025477409362793,
-    "sps": 2203.7073854518803
+    "num_episodes": 175,
+    "mean_reward": 115.29129805564881,
+    "mean_length": 4060.48,
+    "loss": 0.14229056239128113,
+    "sps": 2107.3760666985922
   },
   {
     "update": 340,
     "global_step": 696320,
-    "num_episodes": 236,
-    "mean_reward": 30.600520968437195,
-    "mean_length": 3419.33,
-    "loss": 0.11422204971313477,
-    "sps": 1223.5079096329264
+    "num_episodes": 175,
+    "mean_reward": 115.29129805564881,
+    "mean_length": 4060.48,
+    "loss": 0.36735689640045166,
+    "sps": 2157.1693383290976
   },
   {
     "update": 345,
     "global_step": 706560,
-    "num_episodes": 244,
-    "mean_reward": 29.116036429405213,
-    "mean_length": 3431.39,
-    "loss": 0.3314732015132904,
-    "sps": 604.6873121204968
+    "num_episodes": 189,
+    "mean_reward": 109.84256043434144,
+    "mean_length": 3863.03,
+    "loss": 0.7830713987350464,
+    "sps": 381.54779046834835
   },
   {
     "update": 350,
     "global_step": 716800,
-    "num_episodes": 248,
-    "mean_reward": 27.764892373085022,
-    "mean_length": 3338.43,
-    "loss": -0.08348219841718674,
-    "sps": 2334.652125215495
+    "num_episodes": 193,
+    "mean_reward": 109.65507596015931,
+    "mean_length": 3901.01,
+    "loss": 9.749893188476562,
+    "sps": 258.15120412826286
   },
   {
     "update": 355,
     "global_step": 727040,
-    "num_episodes": 248,
-    "mean_reward": 27.764892373085022,
-    "mean_length": 3338.43,
-    "loss": -0.011928500607609749,
-    "sps": 2266.898774941684
+    "num_episodes": 197,
+    "mean_reward": 109.78353757858277,
+    "mean_length": 3921.28,
+    "loss": 0.27189120650291443,
+    "sps": 2234.7686845905673
   },
   {
     "update": 360,
     "global_step": 737280,
-    "num_episodes": 251,
-    "mean_reward": 27.271975588798522,
-    "mean_length": 3335.03,
-    "loss": 0.16195061802864075,
-    "sps": 511.5670473274587
+    "num_episodes": 198,
+    "mean_reward": 110.03232297897338,
+    "mean_length": 3921.28,
+    "loss": 0.27451807260513306,
+    "sps": 2208.748818348954
   },
   {
     "update": 365,
     "global_step": 747520,
-    "num_episodes": 261,
-    "mean_reward": 25.41567024707794,
-    "mean_length": 3238.35,
-    "loss": -0.17278775572776794,
-    "sps": 810.6621085529184
+    "num_episodes": 198,
+    "mean_reward": 110.03232297897338,
+    "mean_length": 3921.28,
+    "loss": 0.3887019753456116,
+    "sps": 2179.671400488869
   },
   {
     "update": 370,
     "global_step": 757760,
-    "num_episodes": 274,
-    "mean_reward": 21.383243069648742,
-    "mean_length": 2943.09,
-    "loss": -0.017205242067575455,
-    "sps": 619.3760616179322
+    "num_episodes": 200,
+    "mean_reward": 110.02969996452332,
+    "mean_length": 3920.77,
+    "loss": 1.3407719135284424,
+    "sps": 190.58035959703773
   },
   {
     "update": 375,
     "global_step": 768000,
-    "num_episodes": 274,
-    "mean_reward": 21.383243069648742,
-    "mean_length": 2943.09,
-    "loss": 0.08961383998394012,
-    "sps": 2236.482803815745
+    "num_episodes": 205,
+    "mean_reward": 113.28294358253478,
+    "mean_length": 4042.39,
+    "loss": 2.234469175338745,
+    "sps": 165.74069059867267
   },
   {
     "update": 380,
     "global_step": 778240,
-    "num_episodes": 274,
-    "mean_reward": 21.383243069648742,
-    "mean_length": 2943.09,
-    "loss": 4.401951313018799,
-    "sps": 2147.399362575019
+    "num_episodes": 209,
+    "mean_reward": 112.94422784805298,
+    "mean_length": 4057.27,
+    "loss": 0.2318531721830368,
+    "sps": 2224.56366677181
   },
   {
     "update": 385,
     "global_step": 788480,
-    "num_episodes": 278,
-    "mean_reward": 22.28250171661377,
-    "mean_length": 3028.1,
-    "loss": 1.4729293584823608,
-    "sps": 2300.4546825723296
+    "num_episodes": 209,
+    "mean_reward": 112.94422784805298,
+    "mean_length": 4057.27,
+    "loss": 0.2268732637166977,
+    "sps": 2253.631967291392
   },
   {
     "update": 390,
     "global_step": 798720,
-    "num_episodes": 280,
-    "mean_reward": 21.386626567840576,
-    "mean_length": 3127.16,
-    "loss": 0.06943056732416153,
-    "sps": 2341.8789187579932
+    "num_episodes": 210,
+    "mean_reward": 115.53032508850097,
+    "mean_length": 4153.44,
+    "loss": 0.06943273544311523,
+    "sps": 2241.199985702131
   },
   {
     "update": 395,
     "global_step": 808960,
-    "num_episodes": 280,
-    "mean_reward": 21.386626567840576,
-    "mean_length": 3127.16,
-    "loss": 0.929630696773529,
-    "sps": 2232.925422842351
+    "num_episodes": 211,
+    "mean_reward": 118.63075592041015,
+    "mean_length": 4252.74,
+    "loss": 0.21790897846221924,
+    "sps": 2233.8736866140016
   },
   {
     "update": 400,
     "global_step": 819200,
-    "num_episodes": 280,
-    "mean_reward": 21.386626567840576,
-    "mean_length": 3127.16,
-    "loss": -0.035214584320783615,
-    "sps": 2187.4650402736006
+    "num_episodes": 218,
+    "mean_reward": 112.67819341659546,
+    "mean_length": 4053.49,
+    "loss": 0.28217822313308716,
+    "sps": 2245.8419819824057
   },
   {
     "update": 405,
     "global_step": 829440,
-    "num_episodes": 284,
-    "mean_reward": 21.765649213790894,
-    "mean_length": 3031.31,
-    "loss": 3.6039881706237793,
-    "sps": 2344.35431842087
+    "num_episodes": 218,
+    "mean_reward": 112.67819341659546,
+    "mean_length": 4053.49,
+    "loss": 0.07389844954013824,
+    "sps": 2093.830293640606
   },
   {
     "update": 410,
     "global_step": 839680,
-    "num_episodes": 288,
-    "mean_reward": 23.637630491256715,
-    "mean_length": 3126.66,
-    "loss": 0.8354597091674805,
-    "sps": 1416.6940593723025
+    "num_episodes": 219,
+    "mean_reward": 112.67819341659546,
+    "mean_length": 4053.49,
+    "loss": 0.19773584604263306,
+    "sps": 2089.0640172885346
   },
   {
     "update": 415,
     "global_step": 849920,
-    "num_episodes": 288,
-    "mean_reward": 23.637630491256715,
-    "mean_length": 3126.66,
-    "loss": 1.1654760837554932,
-    "sps": 2444.537765977125
+    "num_episodes": 221,
+    "mean_reward": 115.57382575035095,
+    "mean_length": 4142.16,
+    "loss": 0.2218392789363861,
+    "sps": 2037.8682435790647
   },
   {
     "update": 420,
     "global_step": 860160,
-    "num_episodes": 288,
-    "mean_reward": 23.637630491256715,
-    "mean_length": 3126.66,
-    "loss": 0.7069441676139832,
-    "sps": 2402.946945101056
+    "num_episodes": 225,
+    "mean_reward": 118.26686740875245,
+    "mean_length": 4220.72,
+    "loss": 0.16683566570281982,
+    "sps": 2062.011715868465
   },
   {
     "update": 425,
     "global_step": 870400,
-    "num_episodes": 296,
-    "mean_reward": 26.219414005279543,
-    "mean_length": 3426.79,
-    "loss": 1.0991233587265015,
-    "sps": 1584.3855114127348
+    "num_episodes": 225,
+    "mean_reward": 118.26686740875245,
+    "mean_length": 4220.72,
+    "loss": 0.3665807843208313,
+    "sps": 2085.879289076417
   },
   {
     "update": 430,
     "global_step": 880640,
-    "num_episodes": 302,
-    "mean_reward": 24.143256397247313,
-    "mean_length": 3329.74,
-    "loss": 0.5094761252403259,
-    "sps": 1460.1652484484139
+    "num_episodes": 227,
+    "mean_reward": 115.4131219291687,
+    "mean_length": 4123.01,
+    "loss": 0.2211420238018036,
+    "sps": 1319.1510855667186
   },
   {
     "update": 435,
     "global_step": 890880,
-    "num_episodes": 302,
-    "mean_reward": 24.143256397247313,
-    "mean_length": 3329.74,
-    "loss": 1.6765072345733643,
-    "sps": 2365.579425966714
+    "num_episodes": 228,
+    "mean_reward": 118.51452213287354,
+    "mean_length": 4220.72,
+    "loss": 0.01409757137298584,
+    "sps": 2249.26043334859
   },
   {
     "update": 440,
     "global_step": 901120,
-    "num_episodes": 302,
-    "mean_reward": 24.143256397247313,
-    "mean_length": 3329.74,
-    "loss": -0.044437870383262634,
-    "sps": 2282.7314942544617
+    "num_episodes": 232,
+    "mean_reward": 121.46894996643067,
+    "mean_length": 4313.96,
+    "loss": 0.34094715118408203,
+    "sps": 2235.574796161653
   },
   {
     "update": 445,
     "global_step": 911360,
-    "num_episodes": 308,
-    "mean_reward": 23.534629821777344,
-    "mean_length": 3238.0,
-    "loss": 0.4068017303943634,
-    "sps": 865.914540942848
+    "num_episodes": 232,
+    "mean_reward": 121.46894996643067,
+    "mean_length": 4313.96,
+    "loss": 0.03650672733783722,
+    "sps": 2254.642878384582
   },
   {
     "update": 450,
     "global_step": 921600,
-    "num_episodes": 314,
-    "mean_reward": 23.246068153381348,
-    "mean_length": 3138.43,
-    "loss": 0.32406821846961975,
-    "sps": 830.3529260738604
+    "num_episodes": 233,
+    "mean_reward": 122.21637367248535,
+    "mean_length": 4313.96,
+    "loss": 0.4388778805732727,
+    "sps": 2230.3176453482333
   },
   {
     "update": 455,
     "global_step": 931840,
-    "num_episodes": 316,
-    "mean_reward": 22.337194681167603,
-    "mean_length": 3057.38,
-    "loss": 0.21466466784477234,
-    "sps": 2411.145412824079
+    "num_episodes": 235,
+    "mean_reward": 122.47082150459289,
+    "mean_length": 4315.26,
+    "loss": 0.04285623878240585,
+    "sps": 2245.5355177671836
   },
   {
     "update": 460,
     "global_step": 942080,
-    "num_episodes": 316,
-    "mean_reward": 22.337194681167603,
-    "mean_length": 3057.38,
-    "loss": 4.0019731521606445,
-    "sps": 2372.8288290677337
+    "num_episodes": 241,
+    "mean_reward": 125.87507015228272,
+    "mean_length": 4387.15,
+    "loss": 0.18071885406970978,
+    "sps": 2186.650382997658
   },
   {
     "update": 465,
     "global_step": 952320,
-    "num_episodes": 321,
-    "mean_reward": 22.935798473358155,
-    "mean_length": 3155.68,
-    "loss": 3.8567330837249756,
-    "sps": 796.2439424516102
+    "num_episodes": 241,
+    "mean_reward": 125.87507015228272,
+    "mean_length": 4387.15,
+    "loss": 0.15905889868736267,
+    "sps": 2246.1825959008293
   },
   {
     "update": 470,
     "global_step": 962560,
-    "num_episodes": 326,
-    "mean_reward": 23.01468356132507,
-    "mean_length": 3080.66,
-    "loss": 0.04481935873627663,
-    "sps": 2363.29177191282
+    "num_episodes": 243,
+    "mean_reward": 125.64015349388123,
+    "mean_length": 4385.41,
+    "loss": 0.2828221917152405,
+    "sps": 2237.777989892148
   },
   {
     "update": 475,
     "global_step": 972800,
-    "num_episodes": 329,
-    "mean_reward": 24.00417799949646,
-    "mean_length": 3081.1,
-    "loss": -0.09712222218513489,
-    "sps": 2323.0109350014427
+    "num_episodes": 244,
+    "mean_reward": 128.7428973865509,
+    "mean_length": 4484.47,
+    "loss": -0.001385476440191269,
+    "sps": 2224.056236169126
   },
   {
     "update": 480,
     "global_step": 983040,
-    "num_episodes": 329,
-    "mean_reward": 24.00417799949646,
-    "mean_length": 3081.1,
-    "loss": 0.29562610387802124,
-    "sps": 2249.6786763931746
+    "num_episodes": 247,
+    "mean_reward": 134.21521110534667,
+    "mean_length": 4676.7,
+    "loss": 0.6856433153152466,
+    "sps": 166.9800451494931
   },
   {
     "update": 485,
     "global_step": 993280,
-    "num_episodes": 330,
-    "mean_reward": 24.60798607826233,
-    "mean_length": 3177.27,
-    "loss": 0.02162332460284233,
-    "sps": 2222.1627608215495
+    "num_episodes": 248,
+    "mean_reward": 131.0992687225342,
+    "mean_length": 4598.8,
+    "loss": 0.22580231726169586,
+    "sps": 2205.4840180784313
   },
   {
     "update": 490,
     "global_step": 1003520,
-    "num_episodes": 334,
-    "mean_reward": 25.107961702346802,
-    "mean_length": 3177.27,
-    "loss": 0.30668944120407104,
-    "sps": 2336.9965399128855
+    "num_episodes": 253,
+    "mean_reward": 130.85596141815185,
+    "mean_length": 4590.29,
+    "loss": 15.813247680664062,
+    "sps": 1554.4720078900461
   },
   {
     "update": 495,
     "global_step": 1013760,
-    "num_episodes": 337,
-    "mean_reward": 25.60792893409729,
-    "mean_length": 3175.92,
-    "loss": 0.40231630206108093,
-    "sps": 1175.8081438816857
+    "num_episodes": 262,
+    "mean_reward": 118.73081938743591,
+    "mean_length": 4225.92,
+    "loss": 3.043635368347168,
+    "sps": 364.1160359745992
   },
   {
     "update": 500,
     "global_step": 1024000,
-    "num_episodes": 337,
-    "mean_reward": 25.60792893409729,
-    "mean_length": 3175.92,
-    "loss": 0.33998745679855347,
-    "sps": 2419.5867438311334
+    "num_episodes": 266,
+    "mean_reward": 112.27171317100525,
+    "mean_length": 4026.32,
+    "loss": 1.5756291151046753,
+    "sps": 2216.9506893762605
   },
   {
     "update": 505,
     "global_step": 1034240,
-    "num_episodes": 340,
-    "mean_reward": 25.602466859817504,
-    "mean_length": 3171.49,
-    "loss": 0.09274712204933167,
-    "sps": 2409.938615394154
+    "num_episodes": 271,
+    "mean_reward": 106.31812086105347,
+    "mean_length": 3831.41,
+    "loss": 0.8373554348945618,
+    "sps": 2230.0889295621437
   },
   {
     "update": 510,
     "global_step": 1044480,
-    "num_episodes": 346,
-    "mean_reward": 25.201329984664916,
-    "mean_length": 3263.94,
-    "loss": 0.2879057228565216,
-    "sps": 976.7251772331928
+    "num_episodes": 271,
+    "mean_reward": 106.31812086105347,
+    "mean_length": 3831.41,
+    "loss": 0.1685715615749359,
+    "sps": 2250.612331563048
   },
   {
     "update": 515,
     "global_step": 1054720,
-    "num_episodes": 347,
-    "mean_reward": 25.555161905288696,
-    "mean_length": 3361.65,
-    "loss": 0.39604732394218445,
-    "sps": 2406.938365656791
+    "num_episodes": 279,
+    "mean_reward": 94.40078766822815,
+    "mean_length": 3437.38,
+    "loss": 0.3164912760257721,
+    "sps": 755.1427540116495
   },
   {
     "update": 520,
     "global_step": 1064960,
-    "num_episodes": 347,
-    "mean_reward": 25.555161905288696,
-    "mean_length": 3361.65,
-    "loss": 0.03350641578435898,
-    "sps": 2384.9415454427835
+    "num_episodes": 284,
+    "mean_reward": 97.14825866699219,
+    "mean_length": 3536.28,
+    "loss": 2.4807233810424805,
+    "sps": 2229.202303336277
   },
   {
     "update": 525,
     "global_step": 1075200,
-    "num_episodes": 348,
-    "mean_reward": 26.658946142196655,
-    "mean_length": 3459.36,
-    "loss": 0.8593693971633911,
-    "sps": 2415.202845406159
+    "num_episodes": 288,
+    "mean_reward": 100.25763621330262,
+    "mean_length": 3636.97,
+    "loss": 7.4268412590026855,
+    "sps": 2240.4780501902205
   },
   {
     "update": 530,
     "global_step": 1085440,
-    "num_episodes": 355,
-    "mean_reward": 27.258036041259764,
-    "mean_length": 3553.8,
-    "loss": 0.4878949820995331,
-    "sps": 710.1710154431543
+    "num_episodes": 288,
+    "mean_reward": 100.25763621330262,
+    "mean_length": 3636.97,
+    "loss": 13.631644248962402,
+    "sps": 2167.9135230629645
   },
   {
     "update": 535,
     "global_step": 1095680,
-    "num_episodes": 368,
-    "mean_reward": 28.800746297836305,
-    "mean_length": 3371.19,
-    "loss": 0.21109388768672943,
-    "sps": 532.1724281533873
+    "num_episodes": 289,
+    "mean_reward": 103.11298480987548,
+    "mean_length": 3736.54,
+    "loss": 16.333158493041992,
+    "sps": 2159.176669252336
   },
   {
     "update": 540,
     "global_step": 1105920,
-    "num_episodes": 371,
-    "mean_reward": 29.045905199050903,
-    "mean_length": 3369.84,
-    "loss": 0.14603528380393982,
-    "sps": 2335.2550876461346
+    "num_episodes": 292,
+    "mean_reward": 103.19977847099304,
+    "mean_length": 3834.19,
+    "loss": 1.755424976348877,
+    "sps": 2140.8384853415
   },
   {
     "update": 545,
     "global_step": 1116160,
-    "num_episodes": 373,
-    "mean_reward": 29.899697828292847,
-    "mean_length": 3466.01,
-    "loss": -0.04600970447063446,
-    "sps": 2305.921805837758
+    "num_episodes": 293,
+    "mean_reward": 106.40395077705384,
+    "mean_length": 3909.42,
+    "loss": 2.900822639465332,
+    "sps": 175.96426214417178
   },
   {
     "update": 550,
     "global_step": 1126400,
-    "num_episodes": 374,
-    "mean_reward": 31.003479852676392,
-    "mean_length": 3565.07,
-    "loss": 0.11138609051704407,
-    "sps": 2400.3530395714765
+    "num_episodes": 296,
+    "mean_reward": 106.08349679946899,
+    "mean_length": 3927.6,
+    "loss": 3.329728126525879,
+    "sps": 162.40274463106428
   },
   {
     "update": 555,
     "global_step": 1136640,
-    "num_episodes": 377,
-    "mean_reward": 31.149640073776244,
-    "mean_length": 3467.36,
-    "loss": 0.35408008098602295,
-    "sps": 1863.2471573876685
+    "num_episodes": 300,
+    "mean_reward": 100.18753468513489,
+    "mean_length": 3833.08,
+    "loss": 0.3283099830150604,
+    "sps": 2190.1276229463892
   },
   {
     "update": 560,
     "global_step": 1146880,
-    "num_episodes": 378,
-    "mean_reward": 31.50347306251526,
-    "mean_length": 3563.53,
-    "loss": 0.03238815441727638,
-    "sps": 2410.263871749416
+    "num_episodes": 307,
+    "mean_reward": 100.06094589233399,
+    "mean_length": 3798.23,
+    "loss": 3.218208074569702,
+    "sps": 196.6633145580635
   },
   {
     "update": 565,
     "global_step": 1157120,
-    "num_episodes": 379,
-    "mean_reward": 31.503473825454712,
-    "mean_length": 3563.53,
-    "loss": 1.2669872045516968,
-    "sps": 2435.9597448206396
+    "num_episodes": 308,
+    "mean_reward": 99.96278611183166,
+    "mean_length": 3815.32,
+    "loss": 0.05826599895954132,
+    "sps": 2198.7480074814316
   },
   {
     "update": 570,
     "global_step": 1167360,
-    "num_episodes": 381,
-    "mean_reward": 30.14463397026062,
-    "mean_length": 3465.82,
-    "loss": 1.1442028284072876,
-    "sps": 2364.850671069342
+    "num_episodes": 308,
+    "mean_reward": 99.96278611183166,
+    "mean_length": 3815.32,
+    "loss": 0.19391702115535736,
+    "sps": 2247.8319226488716
   },
   {
     "update": 575,
     "global_step": 1177600,
-    "num_episodes": 387,
-    "mean_reward": 27.80320453643799,
-    "mean_length": 3366.44,
-    "loss": 0.08564125001430511,
-    "sps": 566.4595364452883
+    "num_episodes": 311,
+    "mean_reward": 97.06219799995422,
+    "mean_length": 3719.15,
+    "loss": 0.5620024800300598,
+    "sps": 2199.015937226961
   },
   {
     "update": 580,
     "global_step": 1187840,
-    "num_episodes": 389,
-    "mean_reward": 27.20208529472351,
-    "mean_length": 3266.87,
-    "loss": 0.14057211577892303,
-    "sps": 2297.6598442501095
+    "num_episodes": 316,
+    "mean_reward": 96.72605070114136,
+    "mean_length": 3721.99,
+    "loss": 9.720523834228516,
+    "sps": 1486.7675112343536
   },
   {
     "update": 585,
     "global_step": 1198080,
-    "num_episodes": 396,
-    "mean_reward": 26.080838441848755,
-    "mean_length": 3169.09,
-    "loss": 0.0875428318977356,
-    "sps": 2440.645435446852
+    "num_episodes": 318,
+    "mean_reward": 96.66609871864318,
+    "mean_length": 3723.94,
+    "loss": 0.6666443943977356,
+    "sps": 2234.009052616905
   },
   {
     "update": 590,
     "global_step": 1208320,
-    "num_episodes": 403,
-    "mean_reward": 24.67230319023132,
-    "mean_length": 3069.72,
-    "loss": -0.005318094044923782,
-    "sps": 2415.1756827351583
+    "num_episodes": 318,
+    "mean_reward": 96.66609871864318,
+    "mean_length": 3723.94,
+    "loss": 0.2884657084941864,
+    "sps": 2239.9481475435614
   },
   {
     "update": 595,
     "global_step": 1218560,
-    "num_episodes": 406,
-    "mean_reward": 23.778743057250978,
-    "mean_length": 3164.03,
-    "loss": 0.07221215218305588,
-    "sps": 1535.4284659165353
+    "num_episodes": 320,
+    "mean_reward": 96.41710000038147,
+    "mean_length": 3723.94,
+    "loss": 0.8150730133056641,
+    "sps": 2242.7699730761115
   },
   {
     "update": 600,
     "global_step": 1228800,
-    "num_episodes": 413,
-    "mean_reward": 22.0106067276001,
-    "mean_length": 2967.58,
-    "loss": 0.35672813653945923,
-    "sps": 2410.0731702663725
+    "num_episodes": 322,
+    "mean_reward": 101.80320601463318,
+    "mean_length": 3725.16,
+    "loss": 0.8642079830169678,
+    "sps": 285.7807108395187
   },
   {
     "update": 605,
     "global_step": 1239040,
-    "num_episodes": 418,
-    "mean_reward": 20.706662797927855,
-    "mean_length": 2849.49,
-    "loss": 0.1961039900779724,
-    "sps": 2444.0300319234293
+    "num_episodes": 326,
+    "mean_reward": 98.85420690536499,
+    "mean_length": 3650.4,
+    "loss": 0.6339423060417175,
+    "sps": 1850.348648405877
   },
   {
     "update": 610,
     "global_step": 1249280,
-    "num_episodes": 420,
-    "mean_reward": 21.058296146392824,
-    "mean_length": 2949.06,
-    "loss": 0.20069828629493713,
-    "sps": 2409.835173874158
+    "num_episodes": 326,
+    "mean_reward": 98.85420690536499,
+    "mean_length": 3650.4,
+    "loss": 0.5616691708564758,
+    "sps": 2222.6578253029397
   },
   {
     "update": 615,
     "global_step": 1259520,
-    "num_episodes": 420,
-    "mean_reward": 21.058296146392824,
-    "mean_length": 2949.06,
-    "loss": 0.6283493041992188,
-    "sps": 2351.264188304028
+    "num_episodes": 331,
+    "mean_reward": 90.45196411609649,
+    "mean_length": 3548.02,
+    "loss": 0.2677087187767029,
+    "sps": 2224.9883354292588
   },
   {
     "update": 620,
     "global_step": 1269760,
-    "num_episodes": 436,
-    "mean_reward": 16.700364379882814,
-    "mean_length": 2538.45,
-    "loss": 0.06421943008899689,
-    "sps": 581.6206601898414
+    "num_episodes": 331,
+    "mean_reward": 90.45196411609649,
+    "mean_length": 3548.02,
+    "loss": 0.05597818270325661,
+    "sps": 2232.8842122446686
   },
   {
     "update": 625,
     "global_step": 1280000,
-    "num_episodes": 437,
-    "mean_reward": 17.554159069061278,
-    "mean_length": 2634.62,
-    "loss": 0.025755397975444794,
-    "sps": 2336.2122156272303
+    "num_episodes": 335,
+    "mean_reward": 89.9470157957077,
+    "mean_length": 3546.72,
+    "loss": 0.4655948281288147,
+    "sps": 314.0543075325823
   },
   {
     "update": 630,
     "global_step": 1290240,
-    "num_episodes": 438,
-    "mean_reward": 17.804676685333252,
-    "mean_length": 2634.62,
-    "loss": -0.039572782814502716,
-    "sps": 2353.5698753913625
+    "num_episodes": 337,
+    "mean_reward": 83.67983005046844,
+    "mean_length": 3359.15,
+    "loss": 0.0969170331954956,
+    "sps": 2181.9774383339013
   },
   {
     "update": 635,
     "global_step": 1300480,
-    "num_episodes": 438,
-    "mean_reward": 17.804676685333252,
-    "mean_length": 2634.62,
-    "loss": -0.0706489086151123,
-    "sps": 2283.6836848035246
+    "num_episodes": 342,
+    "mean_reward": 86.38446100711822,
+    "mean_length": 3465.84,
+    "loss": 0.7292745113372803,
+    "sps": 2124.75987235513
   },
   {
     "update": 640,
     "global_step": 1310720,
-    "num_episodes": 441,
-    "mean_reward": 19.15775371551514,
-    "mean_length": 2733.68,
-    "loss": -0.09373792260885239,
-    "sps": 1474.6357445156093
+    "num_episodes": 342,
+    "mean_reward": 86.38446100711822,
+    "mean_length": 3465.84,
+    "loss": 0.04831065982580185,
+    "sps": 2168.1603087847257
   },
   {
     "update": 645,
     "global_step": 1320960,
-    "num_episodes": 442,
-    "mean_reward": 20.157576084136963,
-    "mean_length": 2733.68,
-    "loss": 0.11351752281188965,
-    "sps": 2334.5626594322757
+    "num_episodes": 343,
+    "mean_reward": 89.47943027019501,
+    "mean_length": 3565.41,
+    "loss": 0.1487177461385727,
+    "sps": 2107.2008167857116
   },
   {
     "update": 650,
     "global_step": 1331200,
-    "num_episodes": 444,
-    "mean_reward": 20.267150926589967,
-    "mean_length": 2833.09,
-    "loss": -0.18438173830509186,
-    "sps": 2236.677306457589
+    "num_episodes": 344,
+    "mean_reward": 89.2304312467575,
+    "mean_length": 3565.41,
+    "loss": 0.12002412974834442,
+    "sps": 2127.214484781757
   },
   {
     "update": 655,
     "global_step": 1341440,
-    "num_episodes": 444,
-    "mean_reward": 20.267150926589967,
-    "mean_length": 2833.09,
-    "loss": -0.07986398041248322,
-    "sps": 2210.239531580035
+    "num_episodes": 347,
+    "mean_reward": 88.98148257732392,
+    "mean_length": 3565.41,
+    "loss": 0.1547270268201828,
+    "sps": 2165.6753206938283
   },
   {
     "update": 660,
     "global_step": 1351680,
-    "num_episodes": 447,
-    "mean_reward": 21.873516216278077,
-    "mean_length": 2929.77,
-    "loss": -0.05535223335027695,
-    "sps": 2385.7304437353546
+    "num_episodes": 347,
+    "mean_reward": 88.98148257732392,
+    "mean_length": 3565.41,
+    "loss": 0.1951214224100113,
+    "sps": 2229.5540914965095
   },
   {
     "update": 665,
     "global_step": 1361920,
-    "num_episodes": 449,
-    "mean_reward": 21.519568309783935,
-    "mean_length": 2830.71,
-    "loss": 0.10434924066066742,
-    "sps": 2330.8125431773424
+    "num_episodes": 350,
+    "mean_reward": 88.69014154911041,
+    "mean_length": 3547.65,
+    "loss": 0.02001141756772995,
+    "sps": 2220.3189922208117
   },
   {
     "update": 670,
     "global_step": 1372160,
-    "num_episodes": 451,
-    "mean_reward": 22.623257703781128,
-    "mean_length": 2927.07,
-    "loss": 0.29875361919403076,
-    "sps": 2388.3964178785764
+    "num_episodes": 353,
+    "mean_reward": 91.77625680446624,
+    "mean_length": 3645.53,
+    "loss": 0.1334764063358307,
+    "sps": 2217.9524326914734
   },
   {
     "update": 675,
     "global_step": 1382400,
-    "num_episodes": 451,
-    "mean_reward": 22.623257703781128,
-    "mean_length": 2927.07,
-    "loss": 0.01194491982460022,
-    "sps": 2360.8138719071453
+    "num_episodes": 356,
+    "mean_reward": 94.39934126377106,
+    "mean_length": 3746.85,
+    "loss": 1.491716742515564,
+    "sps": 115.85030478370321
   },
   {
     "update": 680,
     "global_step": 1392640,
-    "num_episodes": 454,
-    "mean_reward": 23.234625358581543,
-    "mean_length": 3026.64,
-    "loss": 0.1673501431941986,
-    "sps": 1557.6244924988657
+    "num_episodes": 362,
+    "mean_reward": 94.36987939357758,
+    "mean_length": 3759.39,
+    "loss": 0.08460421860218048,
+    "sps": 466.5453273147261
   },
   {
     "update": 685,
     "global_step": 1402880,
-    "num_episodes": 458,
-    "mean_reward": 23.841174821853638,
-    "mean_length": 3124.86,
-    "loss": 0.026581741869449615,
-    "sps": 2296.8415092422506
+    "num_episodes": 370,
+    "mean_reward": 91.42453453540801,
+    "mean_length": 3664.28,
+    "loss": 0.06896203756332397,
+    "sps": 2211.12877876806
   },
   {
     "update": 690,
     "global_step": 1413120,
-    "num_episodes": 459,
-    "mean_reward": 23.949846239089965,
-    "mean_length": 3221.03,
-    "loss": 1.637216329574585,
-    "sps": 413.9001244167228
+    "num_episodes": 371,
+    "mean_reward": 94.27827970981598,
+    "mean_length": 3763.34,
+    "loss": 0.08465250581502914,
+    "sps": 2224.3649291369943
   },
   {
     "update": 695,
     "global_step": 1423360,
-    "num_episodes": 465,
-    "mean_reward": 24.109339590072633,
-    "mean_length": 3246.44,
-    "loss": 0.4041682779788971,
-    "sps": 1973.9302703248002
+    "num_episodes": 372,
+    "mean_reward": 94.03031706333161,
+    "mean_length": 3763.34,
+    "loss": 0.12509916722774506,
+    "sps": 2233.8556777814188
   },
   {
     "update": 700,
     "global_step": 1433600,
-    "num_episodes": 466,
-    "mean_reward": 24.21024447441101,
-    "mean_length": 3344.24,
-    "loss": 0.13472315669059753,
-    "sps": 2279.0860535321326
+    "num_episodes": 372,
+    "mean_reward": 94.03031706333161,
+    "mean_length": 3763.34,
+    "loss": 0.05390219762921333,
+    "sps": 2225.4858130028433
   },
   {
     "update": 705,
     "global_step": 1443840,
-    "num_episodes": 468,
-    "mean_reward": 24.856354351043702,
-    "mean_length": 3440.41,
-    "loss": 0.6666802763938904,
-    "sps": 2315.9586381196755
+    "num_episodes": 378,
+    "mean_reward": 100.22330937862397,
+    "mean_length": 3957.32,
+    "loss": 1.2421371936798096,
+    "sps": 176.73331940579132
   },
   {
     "update": 710,
     "global_step": 1454080,
-    "num_episodes": 468,
-    "mean_reward": 24.856354351043702,
-    "mean_length": 3440.41,
-    "loss": -0.15654346346855164,
-    "sps": 2242.2811278107365
+    "num_episodes": 380,
+    "mean_reward": 100.36302034854889,
+    "mean_length": 3980.35,
+    "loss": 0.06020417809486389,
+    "sps": 2151.8459777584108
   },
   {
     "update": 715,
     "global_step": 1464320,
-    "num_episodes": 477,
-    "mean_reward": 24.00254797935486,
-    "mean_length": 3342.7,
-    "loss": -0.14158941805362701,
-    "sps": 2301.1874356888225
+    "num_episodes": 383,
+    "mean_reward": 103.226567196846,
+    "mean_length": 4079.78,
+    "loss": 0.03555985540151596,
+    "sps": 779.0322633363131
   },
   {
     "update": 720,
     "global_step": 1474560,
-    "num_episodes": 480,
-    "mean_reward": 24.0372780418396,
-    "mean_length": 3146.96,
-    "loss": 0.10493358224630356,
-    "sps": 843.3732587375952
+    "num_episodes": 385,
+    "mean_reward": 100.37291463375091,
+    "mean_length": 3983.61,
+    "loss": 0.42947918176651,
+    "sps": 1976.0163143357508
   },
   {
     "update": 725,
     "global_step": 1484800,
-    "num_episodes": 481,
-    "mean_reward": 23.896179141998292,
-    "mean_length": 3244.67,
-    "loss": 0.01014188677072525,
-    "sps": 2387.5931465826584
+    "num_episodes": 387,
+    "mean_reward": 102.8877759027481,
+    "mean_length": 4082.64,
+    "loss": 0.3529297709465027,
+    "sps": 2149.9991470019922
   },
   {
     "update": 730,
     "global_step": 1495040,
-    "num_episodes": 481,
-    "mean_reward": 23.896179141998292,
-    "mean_length": 3244.67,
-    "loss": 1.5800291299819946,
-    "sps": 2199.482155508925
+    "num_episodes": 393,
+    "mean_reward": 96.80677964687348,
+    "mean_length": 3770.28,
+    "loss": 0.5400058627128601,
+    "sps": 182.2636316464271
   },
   {
     "update": 735,
     "global_step": 1505280,
-    "num_episodes": 486,
-    "mean_reward": 23.005251121520995,
-    "mean_length": 3344.42,
-    "loss": 2.476745128631592,
-    "sps": 555.9822817000502
+    "num_episodes": 397,
+    "mean_reward": 99.80784844875336,
+    "mean_length": 3841.74,
+    "loss": 0.07788175344467163,
+    "sps": 1880.7298583574118
   },
   {
     "update": 740,
     "global_step": 1515520,
-    "num_episodes": 487,
-    "mean_reward": 22.755260047912596,
-    "mean_length": 3344.42,
-    "loss": 0.043340131640434265,
-    "sps": 2358.713519842451
+    "num_episodes": 400,
+    "mean_reward": 102.611374335289,
+    "mean_length": 3837.4,
+    "loss": 0.12514562904834747,
+    "sps": 2199.6995663818816
   },
   {
     "update": 745,
     "global_step": 1525760,
-    "num_episodes": 489,
-    "mean_reward": 23.35637879371643,
-    "mean_length": 3442.64,
-    "loss": 0.8548387885093689,
-    "sps": 2384.7462230357132
+    "num_episodes": 402,
+    "mean_reward": 102.8225530385971,
+    "mean_length": 3839.8,
+    "loss": 0.1331540048122406,
+    "sps": 505.88516130939627
   },
   {
     "update": 750,
     "global_step": 1536000,
-    "num_episodes": 489,
-    "mean_reward": 23.35637879371643,
-    "mean_length": 3442.64,
-    "loss": 0.06128331273794174,
-    "sps": 2389.02547098786
-  },
-  {
-    "update": 755,
-    "global_step": 1546240,
-    "num_episodes": 495,
-    "mean_reward": 23.45271695137024,
-    "mean_length": 3539.52,
-    "loss": 0.010654259473085403,
-    "sps": 451.1509132304138
-  },
-  {
-    "update": 760,
-    "global_step": 1556480,
-    "num_episodes": 498,
-    "mean_reward": 24.20286971092224,
-    "mean_length": 3541.89,
-    "loss": -0.015714531764388084,
-    "sps": 2272.008455427166
-  },
-  {
-    "update": 765,
-    "global_step": 1566720,
-    "num_episodes": 503,
-    "mean_reward": 24.520383558273316,
-    "mean_length": 3642.43,
-    "loss": 1.2781355381011963,
-    "sps": 2234.132232505493
-  },
-  {
-    "update": 770,
-    "global_step": 1576960,
-    "num_episodes": 503,
-    "mean_reward": 24.520383558273316,
-    "mean_length": 3642.43,
-    "loss": 0.005558963865041733,
-    "sps": 2252.444167541167
-  },
-  {
-    "update": 775,
-    "global_step": 1587200,
-    "num_episodes": 508,
-    "mean_reward": 26.765025606155394,
-    "mean_length": 3642.43,
-    "loss": 1.5083853006362915,
-    "sps": 1001.7898928543148
-  },
-  {
-    "update": 780,
-    "global_step": 1597440,
-    "num_episodes": 512,
-    "mean_reward": 27.61695213317871,
-    "mean_length": 3739.96,
-    "loss": -0.03274097293615341,
-    "sps": 406.1884387297904
-  },
-  {
-    "update": 785,
-    "global_step": 1607680,
-    "num_episodes": 517,
-    "mean_reward": 29.239461097717285,
-    "mean_length": 3749.65,
-    "loss": 0.056141212582588196,
-    "sps": 711.222362259853
-  },
-  {
-    "update": 790,
-    "global_step": 1617920,
-    "num_episodes": 517,
-    "mean_reward": 29.239461097717285,
-    "mean_length": 3749.65,
-    "loss": 0.08803388476371765,
-    "sps": 2290.465987254353
-  },
-  {
-    "update": 795,
-    "global_step": 1628160,
-    "num_episodes": 524,
-    "mean_reward": 28.533955936431884,
-    "mean_length": 3547.65,
-    "loss": 5.9130449295043945,
-    "sps": 1206.367279137959
-  },
-  {
-    "update": 800,
-    "global_step": 1638400,
-    "num_episodes": 528,
-    "mean_reward": 29.898164215087892,
-    "mean_length": 3657.29,
-    "loss": 0.8671517372131348,
-    "sps": 2301.788037202966
+    "num_episodes": 402,
+    "mean_reward": 102.8225530385971,
+    "mean_length": 3839.8,
+    "loss": -0.008329648524522781,
+    "sps": 2268.146789029585
   }
 ]