diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -10,13012 +10,13012 @@
   "log_history": [
     {
       "clip_ratio": 0.0,
-      "completion_length": 483.91668701171875,
+      "completion_length": 1058.8333740234375,
       "epoch": 0.001,
-      "grad_norm": 1.9742541324645637,
+      "grad_norm": 48.31227399851087,
       "kl": 0.0,
       "learning_rate": 1e-08,
-      "loss": -0.0694,
-      "reward": 0.5208333432674408,
-      "reward_std": 0.05689104273915291,
-      "rewards/tag_count_reward": 0.5208333432674408,
+      "loss": 0.4472,
+      "reward": 0.4166666865348816,
+      "reward_std": 0.3704710900783539,
+      "rewards/accuracy_reward": 0.4166666865348816,
       "step": 1
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 461.93751525878906,
+      "completion_length": 1004.3958740234375,
       "epoch": 0.002,
-      "grad_norm": 1.9070621603104843,
+      "grad_norm": 56.02522495457959,
       "kl": 0.0,
       "learning_rate": 2e-08,
-      "loss": 0.0922,
-      "reward": 0.515625,
-      "reward_std": 0.06549893878400326,
-      "rewards/tag_count_reward": 0.515625,
+      "loss": 0.5325,
+      "reward": 0.4791666865348816,
+      "reward_std": 0.4565621316432953,
+      "rewards/accuracy_reward": 0.4791666865348816,
       "step": 2
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 397.56251525878906,
+      "completion_length": 913.0208435058594,
       "epoch": 0.003,
-      "grad_norm": 1.3934674579092003,
-      "kl": 0.00014257431030273438,
+      "grad_norm": 55.61478153115687,
+      "kl": 8.58306884765625e-05,
       "learning_rate": 3e-08,
-      "loss": -0.0461,
-      "reward": 0.5052083432674408,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.5052083432674408,
+      "loss": 0.5392,
+      "reward": 0.5416666865348816,
+      "reward_std": 0.5111141204833984,
+      "rewards/accuracy_reward": 0.5416666865348816,
       "step": 3
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 474.1458435058594,
+      "completion_length": 984.7916870117188,
       "epoch": 0.004,
-      "grad_norm": 2.0218828049837487,
-      "kl": 0.00012159347534179688,
+      "grad_norm": 48.73445287614348,
+      "kl": 8.654594421386719e-05,
       "learning_rate": 4e-08,
-      "loss": -0.1318,
-      "reward": 0.5156250298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.5156250298023224,
+      "loss": 0.4198,
+      "reward": 0.4375000149011612,
+      "reward_std": 0.4565621018409729,
+      "rewards/accuracy_reward": 0.4375000149011612,
       "step": 4
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 456.31251525878906,
+      "completion_length": 916.6458740234375,
       "epoch": 0.005,
-      "grad_norm": 0.008997411780226844,
-      "kl": 0.00015211105346679688,
+      "grad_norm": 35.550347525580804,
+      "kl": 8.869171142578125e-05,
       "learning_rate": 5e-08,
-      "loss": 0.0,
-      "reward": 0.5,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 0.5,
+      "loss": 0.3438,
+      "reward": 0.2916666716337204,
+      "reward_std": 0.3492250144481659,
+      "rewards/accuracy_reward": 0.2916666716337204,
       "step": 5
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 459.9166717529297,
+      "completion_length": 961.3750305175781,
       "epoch": 0.006,
-      "grad_norm": 2.2346740493751307,
-      "kl": 0.00015592575073242188,
+      "grad_norm": 38.81357794221805,
+      "kl": 9.72747802734375e-05,
       "learning_rate": 6e-08,
-      "loss": -0.0896,
-      "reward": 0.515625,
-      "reward_std": 0.04237028583884239,
-      "rewards/tag_count_reward": 0.515625,
+      "loss": 0.4254,
+      "reward": 0.4375000149011612,
+      "reward_std": 0.4936225861310959,
+      "rewards/accuracy_reward": 0.4375000149011612,
       "step": 6
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 397.1041717529297,
+      "completion_length": 999.6042175292969,
       "epoch": 0.007,
-      "grad_norm": 1.460322899370902,
-      "kl": 0.0001239776611328125,
+      "grad_norm": 52.578095293164594,
+      "kl": 9.179115295410156e-05,
       "learning_rate": 7e-08,
-      "loss": 0.0045,
-      "reward": 0.5104166865348816,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.5104166865348816,
+      "loss": 0.4636,
+      "reward": 0.4583333432674408,
+      "reward_std": 0.4470839649438858,
+      "rewards/accuracy_reward": 0.4583333432674408,
       "step": 7
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 457.7708435058594,
+      "completion_length": 748.9166870117188,
       "epoch": 0.008,
-      "grad_norm": 1.6052570950629088,
-      "kl": 0.000202178955078125,
+      "grad_norm": 26.78726420006489,
+      "kl": 0.00011348724365234375,
       "learning_rate": 8e-08,
-      "loss": 0.0522,
-      "reward": 0.5104166865348816,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.5104166865348816,
+      "loss": 0.2077,
+      "reward": 0.4791666865348816,
+      "reward_std": 0.4370592087507248,
+      "rewards/accuracy_reward": 0.4791666865348816,
       "step": 8
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 423.68751525878906,
+      "completion_length": 823.9583435058594,
       "epoch": 0.009,
-      "grad_norm": 1.7356930498415466,
-      "kl": 0.00014591217041015625,
+      "grad_norm": 44.03215120687268,
+      "kl": 9.489059448242188e-05,
       "learning_rate": 9e-08,
-      "loss": -0.0082,
-      "reward": 0.5312500298023224,
-      "reward_std": 0.08474057167768478,
-      "rewards/tag_count_reward": 0.5312500298023224,
+      "loss": 0.3198,
+      "reward": 0.6458333432674408,
+      "reward_std": 0.3749151676893234,
+      "rewards/accuracy_reward": 0.6458333432674408,
       "step": 9
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 433.5208435058594,
+      "completion_length": 879.375,
       "epoch": 0.01,
-      "grad_norm": 1.264090874402394,
-      "kl": 0.00014019012451171875,
+      "grad_norm": 48.26410896388307,
+      "kl": 7.987022399902344e-05,
       "learning_rate": 1e-07,
-      "loss": 0.0134,
-      "reward": 0.5052083432674408,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.5052083432674408,
+      "loss": 0.442,
+      "reward": 0.645833358168602,
+      "reward_std": 0.3823819011449814,
+      "rewards/accuracy_reward": 0.645833358168602,
       "step": 10
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 423.6666717529297,
+      "completion_length": 1001.2083740234375,
       "epoch": 0.011,
-      "grad_norm": 2.0692894517993046,
-      "kl": 0.00016450881958007812,
+      "grad_norm": 46.47504553509871,
+      "kl": 9.679794311523438e-05,
       "learning_rate": 1.0999999999999999e-07,
-      "loss": -0.0564,
-      "reward": 0.5156250298023224,
-      "reward_std": 0.04237028583884239,
-      "rewards/tag_count_reward": 0.5156250298023224,
+      "loss": 0.3603,
+      "reward": 0.4375000149011612,
+      "reward_std": 0.4936225861310959,
+      "rewards/accuracy_reward": 0.4375000149011612,
       "step": 11
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 423.0416717529297,
+      "completion_length": 1003.7708740234375,
       "epoch": 0.012,
-      "grad_norm": 2.2937606521774225,
-      "kl": 0.00015497207641601562,
+      "grad_norm": 65.85111951125275,
+      "kl": 0.00011444091796875,
       "learning_rate": 1.2e-07,
-      "loss": -0.0519,
-      "reward": 0.5416666865348816,
-      "reward_std": 0.10202579200267792,
-      "rewards/tag_count_reward": 0.5416666865348816,
+      "loss": 0.6047,
+      "reward": 0.5000000149011612,
+      "reward_std": 0.4426998794078827,
+      "rewards/accuracy_reward": 0.5000000149011612,
       "step": 12
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 469.0208435058594,
+      "completion_length": 1187.3125610351562,
       "epoch": 0.013,
-      "grad_norm": 2.0091151598756,
-      "kl": 0.00016117095947265625,
+      "grad_norm": 52.498658735012434,
+      "kl": 0.00014019012451171875,
       "learning_rate": 1.3e-07,
-      "loss": -0.0659,
-      "reward": 0.5208333432674408,
-      "reward_std": 0.05689104646444321,
-      "rewards/tag_count_reward": 0.5208333432674408,
+      "loss": 0.4685,
+      "reward": 0.4791666716337204,
+      "reward_std": 0.5167547464370728,
+      "rewards/accuracy_reward": 0.4791666716337204,
       "step": 13
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 487.0208435058594,
+      "completion_length": 1150.625,
       "epoch": 0.014,
-      "grad_norm": 2.972602173974025,
-      "kl": 0.0001373291015625,
+      "grad_norm": 49.159701319819945,
+      "kl": 0.000148773193359375,
       "learning_rate": 1.4e-07,
-      "loss": -0.1794,
-      "reward": 0.5364583730697632,
-      "reward_std": 0.07692287117242813,
-      "rewards/tag_count_reward": 0.5364583730697632,
+      "loss": 0.4708,
+      "reward": 0.3958333432674408,
+      "reward_std": 0.4722275286912918,
+      "rewards/accuracy_reward": 0.3958333432674408,
       "step": 14
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 426.75001525878906,
+      "completion_length": 1131.875,
       "epoch": 0.015,
-      "grad_norm": 1.196878010043536,
-      "kl": 0.00016832351684570312,
+      "grad_norm": 61.43628428083696,
+      "kl": 0.000293731689453125,
       "learning_rate": 1.5e-07,
-      "loss": 0.0144,
-      "reward": 0.5104166865348816,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.5104166865348816,
+      "loss": 0.548,
+      "reward": 0.5,
+      "reward_std": 0.4213048070669174,
+      "rewards/accuracy_reward": 0.5,
       "step": 15
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 458.2708435058594,
+      "completion_length": 1040.9167175292969,
       "epoch": 0.016,
-      "grad_norm": 1.3805638695285072,
-      "kl": 0.00014972686767578125,
+      "grad_norm": 48.91982064204641,
+      "kl": 0.0030517578125,
       "learning_rate": 1.6e-07,
-      "loss": -0.0195,
-      "reward": 0.5052083432674408,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.5052083432674408,
+      "loss": 0.3904,
+      "reward": 0.520833358168602,
+      "reward_std": 0.3258185237646103,
+      "rewards/accuracy_reward": 0.520833358168602,
       "step": 16
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 440.4583435058594,
+      "completion_length": 1018.5625,
       "epoch": 0.017,
-      "grad_norm": 2.336321857820306,
-      "kl": 0.00016546249389648438,
+      "grad_norm": 44.232282849131074,
+      "kl": 0.00333404541015625,
       "learning_rate": 1.7000000000000001e-07,
-      "loss": 0.0661,
-      "reward": 0.5260416865348816,
-      "reward_std": 0.09021097794175148,
-      "rewards/tag_count_reward": 0.5260416865348816,
+      "loss": 0.4107,
+      "reward": 0.3333333432674408,
+      "reward_std": 0.3880225718021393,
+      "rewards/accuracy_reward": 0.3333333432674408,
       "step": 17
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 422.0416717529297,
+      "completion_length": 830.2291870117188,
       "epoch": 0.018,
-      "grad_norm": 1.5675504056835527,
-      "kl": 0.000171661376953125,
+      "grad_norm": 58.279122519017896,
+      "kl": 0.00275421142578125,
       "learning_rate": 1.8e-07,
-      "loss": 0.0444,
-      "reward": 0.5208333432674408,
-      "reward_std": 0.07216878235340118,
-      "rewards/tag_count_reward": 0.5208333432674408,
+      "loss": 0.5708,
+      "reward": 0.5416666865348816,
+      "reward_std": 0.4879819303750992,
+      "rewards/accuracy_reward": 0.5416666865348816,
       "step": 18
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 479.37501525878906,
+      "completion_length": 1141.1666870117188,
       "epoch": 0.019,
-      "grad_norm": 1.8987192317698203,
-      "kl": 0.00018739700317382812,
+      "grad_norm": 46.19284177629288,
+      "kl": 0.005859375,
       "learning_rate": 1.8999999999999998e-07,
-      "loss": -0.035,
-      "reward": 0.5208333432674408,
-      "reward_std": 0.05689104646444321,
-      "rewards/tag_count_reward": 0.5208333432674408,
+      "loss": 0.3975,
+      "reward": 0.3958333432674408,
+      "reward_std": 0.452178031206131,
+      "rewards/accuracy_reward": 0.3958333432674408,
       "step": 19
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 456.3125,
+      "completion_length": 886.4167175292969,
       "epoch": 0.02,
-      "grad_norm": 1.579116890174869,
-      "kl": 0.00020647048950195312,
+      "grad_norm": 52.31653191370914,
+      "kl": 0.00444793701171875,
       "learning_rate": 2e-07,
-      "loss": -0.0064,
-      "reward": 0.515625,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.515625,
+      "loss": 0.4318,
+      "reward": 0.3958333432674408,
+      "reward_std": 0.3973517566919327,
+      "rewards/accuracy_reward": 0.3958333432674408,
       "step": 20
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 452.68751525878906,
+      "completion_length": 968.2500305175781,
       "epoch": 0.021,
-      "grad_norm": 2.2831411081336745,
-      "kl": 0.00019550323486328125,
+      "grad_norm": 61.304049078182494,
+      "kl": 0.0072479248046875,
       "learning_rate": 2.0999999999999997e-07,
-      "loss": -0.0486,
-      "reward": 0.5364583432674408,
-      "reward_std": 0.10278276726603508,
-      "rewards/tag_count_reward": 0.5364583432674408,
+      "loss": 0.4968,
+      "reward": 0.5625000149011612,
+      "reward_std": 0.5010893195867538,
+      "rewards/accuracy_reward": 0.5625000149011612,
       "step": 21
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 420.5625,
+      "completion_length": 972.9375610351562,
       "epoch": 0.022,
-      "grad_norm": 2.2074315098000388,
-      "kl": 0.00019168853759765625,
+      "grad_norm": 64.89348063272188,
+      "kl": 0.041015625,
       "learning_rate": 2.1999999999999998e-07,
-      "loss": -0.0398,
-      "reward": 0.5364583432674408,
-      "reward_std": 0.0992613285779953,
-      "rewards/tag_count_reward": 0.5364583432674408,
+      "loss": 0.5828,
+      "reward": 0.5625000149011612,
+      "reward_std": 0.4871610254049301,
+      "rewards/accuracy_reward": 0.5625000149011612,
       "step": 22
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 414.2083435058594,
+      "completion_length": 969.0208435058594,
       "epoch": 0.023,
-      "grad_norm": 1.3397307174661355,
-      "kl": 0.00022554397583007812,
+      "grad_norm": 72.40115269551475,
+      "kl": 0.0517578125,
       "learning_rate": 2.3e-07,
-      "loss": -0.0109,
-      "reward": 0.5052083432674408,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.5052083432674408,
+      "loss": 0.6572,
+      "reward": 0.5000000298023224,
+      "reward_std": 0.43888531625270844,
+      "rewards/accuracy_reward": 0.5000000298023224,
       "step": 23
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 415.2083435058594,
+      "completion_length": 942.6458740234375,
       "epoch": 0.024,
-      "grad_norm": 2.2052134784327566,
-      "kl": 0.000335693359375,
+      "grad_norm": 64.3958906833254,
+      "kl": 0.04248046875,
       "learning_rate": 2.4e-07,
-      "loss": 0.0173,
-      "reward": 0.53125,
-      "reward_std": 0.07298427075147629,
-      "rewards/tag_count_reward": 0.53125,
+      "loss": 0.5802,
+      "reward": 0.5416666716337204,
+      "reward_std": 0.4482744634151459,
+      "rewards/accuracy_reward": 0.5416666716337204,
       "step": 24
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 386.43751525878906,
+      "completion_length": 701.8750305175781,
       "epoch": 0.025,
-      "grad_norm": 1.179666671856239,
-      "kl": 0.00025177001953125,
+      "grad_norm": 93.21750391540733,
+      "kl": 0.0355224609375,
       "learning_rate": 2.5e-07,
-      "loss": 0.0124,
-      "reward": 0.5104166865348816,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.5104166865348816,
+      "loss": 0.7569,
+      "reward": 0.7916666865348816,
+      "reward_std": 0.3861365020275116,
+      "rewards/accuracy_reward": 0.7916666865348816,
       "step": 25
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 400.9375,
+      "completion_length": 811.5625305175781,
       "epoch": 0.026,
-      "grad_norm": 1.20187270876094,
-      "kl": 0.0008182525634765625,
+      "grad_norm": 63.347054962933946,
+      "kl": 0.051025390625,
       "learning_rate": 2.6e-07,
-      "loss": 0.0233,
-      "reward": 0.5104166865348816,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.5104166865348816,
+      "loss": 0.5119,
+      "reward": 0.6250000298023224,
+      "reward_std": 0.48533496260643005,
+      "rewards/accuracy_reward": 0.6250000298023224,
       "step": 26
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 409.4583435058594,
+      "completion_length": 1236.6666870117188,
       "epoch": 0.027,
-      "grad_norm": 1.6272380189012048,
-      "kl": 0.000553131103515625,
+      "grad_norm": 55.39206210146261,
+      "kl": 0.108642578125,
       "learning_rate": 2.7e-07,
-      "loss": 0.0289,
-      "reward": 0.5156250298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.5156250298023224,
+      "loss": 0.4809,
+      "reward": 0.3750000223517418,
+      "reward_std": 0.41502857208251953,
+      "rewards/accuracy_reward": 0.3750000223517418,
       "step": 27
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 463.31251525878906,
+      "completion_length": 949.3958740234375,
       "epoch": 0.028,
-      "grad_norm": 2.5525370582436557,
-      "kl": 0.0007915496826171875,
+      "grad_norm": 56.775324226755515,
+      "kl": 0.0732421875,
       "learning_rate": 2.8e-07,
-      "loss": -0.0902,
-      "reward": 0.5312500298023224,
-      "reward_std": 0.07679307460784912,
-      "rewards/tag_count_reward": 0.5312500298023224,
+      "loss": 0.4444,
+      "reward": 0.5000000298023224,
+      "reward_std": 0.4056393951177597,
+      "rewards/accuracy_reward": 0.5000000298023224,
       "step": 28
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 430.625,
+      "completion_length": 899.0625305175781,
       "epoch": 0.029,
-      "grad_norm": 2.1748951967997017,
-      "kl": 0.000789642333984375,
+      "grad_norm": 34.78412235020646,
+      "kl": 0.145263671875,
       "learning_rate": 2.9e-07,
-      "loss": -0.036,
-      "reward": 0.5312500298023224,
-      "reward_std": 0.06760228797793388,
-      "rewards/tag_count_reward": 0.5312500298023224,
+      "loss": 0.2844,
+      "reward": 0.583333358168602,
+      "reward_std": 0.3880225718021393,
+      "rewards/accuracy_reward": 0.583333358168602,
       "step": 29
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 397.625,
+      "completion_length": 594.1875305175781,
       "epoch": 0.03,
-      "grad_norm": 2.80101139329913,
-      "kl": 0.001964569091796875,
+      "grad_norm": 40.39941552865415,
+      "kl": 0.2197265625,
       "learning_rate": 3e-07,
-      "loss": -0.0724,
-      "reward": 0.5208333432674408,
-      "reward_std": 0.05689104646444321,
-      "rewards/tag_count_reward": 0.5208333432674408,
+      "loss": 0.3788,
+      "reward": 0.6041666865348816,
+      "reward_std": 0.3139677047729492,
+      "rewards/accuracy_reward": 0.6041666865348816,
       "step": 30
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 420.54168701171875,
+      "completion_length": 836.5833435058594,
       "epoch": 0.031,
-      "grad_norm": 2.070920146055396,
-      "kl": 0.001697540283203125,
+      "grad_norm": 42.822191074836006,
+      "kl": 0.4365234375,
       "learning_rate": 3.1e-07,
-      "loss": -0.1031,
-      "reward": 0.5260416865348816,
-      "reward_std": 0.0631769448518753,
-      "rewards/tag_count_reward": 0.5260416865348816,
+      "loss": 0.3977,
+      "reward": 0.5000000149011612,
+      "reward_std": 0.32399246096611023,
+      "rewards/accuracy_reward": 0.5000000149011612,
       "step": 31
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 518.0,
+      "completion_length": 687.1041870117188,
       "epoch": 0.032,
-      "grad_norm": 2.620019141438342,
-      "kl": 0.00270843505859375,
+      "grad_norm": 51.8484207805597,
+      "kl": 0.263671875,
       "learning_rate": 3.2e-07,
-      "loss": 0.0303,
-      "reward": 0.5572916865348816,
-      "reward_std": 0.12656269967556,
-      "rewards/tag_count_reward": 0.5572916865348816,
+      "loss": 0.4041,
+      "reward": 0.5416666716337204,
+      "reward_std": 0.3647414296865463,
+      "rewards/accuracy_reward": 0.5416666716337204,
       "step": 32
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 448.1666717529297,
+      "completion_length": 839.0,
       "epoch": 0.033,
-      "grad_norm": 2.8682602323119757,
-      "kl": 0.0037078857421875,
+      "grad_norm": 41.58103587946076,
+      "kl": 0.5078125,
       "learning_rate": 3.3e-07,
-      "loss": -0.1747,
-      "reward": 0.5781250298023224,
-      "reward_std": 0.13166124746203423,
-      "rewards/tag_count_reward": 0.5781250298023224,
+      "loss": 0.4167,
+      "reward": 0.5000000149011612,
+      "reward_std": 0.5111140608787537,
+      "rewards/accuracy_reward": 0.5000000149011612,
       "step": 33
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 462.3333435058594,
+      "completion_length": 832.1250305175781,
       "epoch": 0.034,
-      "grad_norm": 2.6216099803692225,
-      "kl": 0.00449371337890625,
+      "grad_norm": 56.43063621515212,
+      "kl": 0.65234375,
       "learning_rate": 3.4000000000000003e-07,
-      "loss": -0.1132,
-      "reward": 0.5729166865348816,
-      "reward_std": 0.14286355674266815,
-      "rewards/tag_count_reward": 0.5729166865348816,
+      "loss": 0.5562,
+      "reward": 0.5416666865348816,
+      "reward_std": 0.3391112834215164,
+      "rewards/accuracy_reward": 0.5416666865348816,
       "step": 34
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 452.81251525878906,
+      "completion_length": 683.7500305175781,
       "epoch": 0.035,
-      "grad_norm": 3.574170032603353,
-      "kl": 0.0053863525390625,
+      "grad_norm": 65.21231718054847,
+      "kl": 0.5458984375,
       "learning_rate": 3.5e-07,
-      "loss": -0.1787,
-      "reward": 0.5468750298023224,
-      "reward_std": 0.12251907214522362,
-      "rewards/tag_count_reward": 0.5468750298023224,
+      "loss": 0.5646,
+      "reward": 0.6458333432674408,
+      "reward_std": 0.42313092947006226,
+      "rewards/accuracy_reward": 0.6458333432674408,
       "step": 35
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 440.6458435058594,
+      "completion_length": 718.5,
       "epoch": 0.036,
-      "grad_norm": 3.1958385995196497,
-      "kl": 0.01177978515625,
+      "grad_norm": 62.14113380682005,
+      "kl": 0.7548828125,
       "learning_rate": 3.6e-07,
-      "loss": -0.0178,
-      "reward": 0.5989583432674408,
-      "reward_std": 0.14811573922634125,
-      "rewards/tag_count_reward": 0.5989583432674408,
+      "loss": 0.5589,
+      "reward": 0.6666666865348816,
+      "reward_std": 0.4213048070669174,
+      "rewards/accuracy_reward": 0.6666666865348816,
       "step": 36
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 490.02085876464844,
+      "completion_length": 547.3125,
       "epoch": 0.037,
-      "grad_norm": 2.7030726759210393,
-      "kl": 0.0063934326171875,
+      "grad_norm": 86.9584702951694,
+      "kl": 0.333984375,
       "learning_rate": 3.7e-07,
-      "loss": 0.0312,
-      "reward": 0.6041666865348816,
-      "reward_std": 0.16423005610704422,
-      "rewards/tag_count_reward": 0.6041666865348816,
+      "loss": 0.629,
+      "reward": 0.8958333432674408,
+      "reward_std": 0.31381870806217194,
+      "rewards/accuracy_reward": 0.8958333432674408,
       "step": 37
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 580.1666870117188,
+      "completion_length": 842.5416870117188,
       "epoch": 0.038,
-      "grad_norm": 2.479505822262608,
-      "kl": 0.01202392578125,
+      "grad_norm": 43.93668405393975,
+      "kl": 1.1015625,
       "learning_rate": 3.7999999999999996e-07,
-      "loss": -0.1006,
-      "reward": 0.5885416865348816,
-      "reward_std": 0.13912813365459442,
-      "rewards/tag_count_reward": 0.5885416865348816,
+      "loss": 0.5944,
+      "reward": 0.5416666865348816,
+      "reward_std": 0.462202787399292,
+      "rewards/accuracy_reward": 0.5416666865348816,
       "step": 38
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 540.9375305175781,
+      "completion_length": 745.5208740234375,
       "epoch": 0.039,
-      "grad_norm": 2.5257873356130585,
-      "kl": 0.011932373046875,
+      "grad_norm": 14.449618472471721,
+      "kl": 2.78515625,
       "learning_rate": 3.8999999999999997e-07,
-      "loss": 0.0596,
-      "reward": 0.6250000298023224,
-      "reward_std": 0.20110102742910385,
-      "rewards/tag_count_reward": 0.6250000298023224,
+      "loss": 0.6483,
+      "reward": 0.6875000298023224,
+      "reward_std": 0.3592497557401657,
+      "rewards/accuracy_reward": 0.6875000298023224,
       "step": 39
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 479.375,
+      "completion_length": 750.9791870117188,
       "epoch": 0.04,
-      "grad_norm": 2.6339110604344533,
-      "kl": 0.01409912109375,
+      "grad_norm": 39.42074735160498,
+      "kl": 5.9375,
       "learning_rate": 4e-07,
-      "loss": -0.156,
-      "reward": 0.6197916865348816,
-      "reward_std": 0.1571667194366455,
-      "rewards/tag_count_reward": 0.6197916865348816,
+      "loss": 0.8943,
+      "reward": 0.7291666865348816,
+      "reward_std": 0.4464483857154846,
+      "rewards/accuracy_reward": 0.7291666865348816,
       "step": 40
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 567.9166870117188,
+      "completion_length": 802.0,
       "epoch": 0.041,
-      "grad_norm": 2.443225018059453,
-      "kl": 0.012725830078125,
+      "grad_norm": 127.14381477849048,
+      "kl": 8.828125,
       "learning_rate": 4.0999999999999994e-07,
-      "loss": -0.0628,
-      "reward": 0.609375,
-      "reward_std": 0.16926345974206924,
-      "rewards/tag_count_reward": 0.609375,
+      "loss": 0.9199,
+      "reward": 0.5208333432674408,
+      "reward_std": 0.4678434431552887,
+      "rewards/accuracy_reward": 0.5208333432674408,
       "step": 41
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 532.1250305175781,
+      "completion_length": 788.5208435058594,
       "epoch": 0.042,
-      "grad_norm": 2.3700512836648335,
-      "kl": 0.019378662109375,
+      "grad_norm": 93.61903821751076,
+      "kl": 8.65625,
       "learning_rate": 4.1999999999999995e-07,
-      "loss": -0.1196,
-      "reward": 0.6510416865348816,
-      "reward_std": 0.15612749755382538,
-      "rewards/tag_count_reward": 0.6510416865348816,
+      "loss": 0.6837,
+      "reward": 0.5625000149011612,
+      "reward_std": 0.3823819011449814,
+      "rewards/accuracy_reward": 0.5625000149011612,
       "step": 42
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 572.8333587646484,
+      "completion_length": 979.5208740234375,
       "epoch": 0.043,
-      "grad_norm": 2.5447339536309275,
-      "kl": 0.013427734375,
+      "grad_norm": 158.93226498926109,
+      "kl": 14.25,
       "learning_rate": 4.2999999999999996e-07,
-      "loss": -0.06,
-      "reward": 0.6354166865348816,
-      "reward_std": 0.1819113940000534,
-      "rewards/tag_count_reward": 0.6354166865348816,
+      "loss": 1.4177,
+      "reward": 0.5625000298023224,
+      "reward_std": 0.5129401683807373,
+      "rewards/accuracy_reward": 0.5625000298023224,
       "step": 43
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 593.5416870117188,
+      "completion_length": 899.2916870117188,
       "epoch": 0.044,
-      "grad_norm": 2.2168144267626757,
-      "kl": 0.0108489990234375,
+      "grad_norm": 90.52473012423707,
+      "kl": 9.59375,
       "learning_rate": 4.3999999999999997e-07,
-      "loss": -0.0839,
-      "reward": 0.6770833432674408,
-      "reward_std": 0.16930782049894333,
-      "rewards/tag_count_reward": 0.6770833432674408,
+      "loss": 0.807,
+      "reward": 0.3333333432674408,
+      "reward_std": 0.4314185380935669,
+      "rewards/accuracy_reward": 0.3333333432674408,
       "step": 44
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 531.3958435058594,
+      "completion_length": 783.7291870117188,
       "epoch": 0.045,
-      "grad_norm": 2.563561843819508,
-      "kl": 0.01080322265625,
+      "grad_norm": 92.60474297049325,
+      "kl": 7.703125,
       "learning_rate": 4.5e-07,
-      "loss": -0.0386,
-      "reward": 0.5833333730697632,
-      "reward_std": 0.13584764301776886,
-      "rewards/tag_count_reward": 0.5833333730697632,
+      "loss": 0.665,
+      "reward": 0.458333358168602,
+      "reward_std": 0.3861365020275116,
+      "rewards/accuracy_reward": 0.458333358168602,
       "step": 45
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 494.41668701171875,
+      "completion_length": 691.4583740234375,
       "epoch": 0.046,
-      "grad_norm": 2.7395597608840148,
-      "kl": 0.016021728515625,
+      "grad_norm": 50.17487340529031,
+      "kl": 5.859375,
       "learning_rate": 4.6e-07,
-      "loss": -0.0226,
-      "reward": 0.6562500298023224,
-      "reward_std": 0.1605742834508419,
-      "rewards/tag_count_reward": 0.6562500298023224,
+      "loss": 0.892,
+      "reward": 0.6666666865348816,
+      "reward_std": 0.38924944400787354,
+      "rewards/accuracy_reward": 0.6666666865348816,
       "step": 46
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 429.7708435058594,
+      "completion_length": 951.0,
       "epoch": 0.047,
-      "grad_norm": 2.9160640815755015,
-      "kl": 0.011383056640625,
+      "grad_norm": 79.11343211201265,
+      "kl": 8.21875,
       "learning_rate": 4.6999999999999995e-07,
-      "loss": -0.0479,
-      "reward": 0.6562500298023224,
-      "reward_std": 0.19549089670181274,
-      "rewards/tag_count_reward": 0.6562500298023224,
+      "loss": 0.9387,
+      "reward": 0.3333333432674408,
+      "reward_std": 0.3509620875120163,
+      "rewards/accuracy_reward": 0.3333333432674408,
       "step": 47
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 414.66668701171875,
+      "completion_length": 993.8125305175781,
       "epoch": 0.048,
-      "grad_norm": 2.41669012498755,
-      "kl": 0.008087158203125,
+      "grad_norm": 67.53012818096163,
+      "kl": 8.015625,
       "learning_rate": 4.8e-07,
-      "loss": 0.0447,
-      "reward": 0.7083333432674408,
-      "reward_std": 0.22293969988822937,
-      "rewards/tag_count_reward": 0.7083333432674408,
+      "loss": 0.9335,
+      "reward": 0.4375000149011612,
+      "reward_std": 0.4414432942867279,
+      "rewards/accuracy_reward": 0.4375000149011612,
       "step": 48
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 489.3958435058594,
+      "completion_length": 975.75,
       "epoch": 0.049,
-      "grad_norm": 2.375274599067812,
-      "kl": 0.007904052734375,
+      "grad_norm": 62.43049430907032,
+      "kl": 7.4375,
       "learning_rate": 4.9e-07,
-      "loss": -0.0285,
-      "reward": 0.6979166865348816,
-      "reward_std": 0.2077343687415123,
-      "rewards/tag_count_reward": 0.6979166865348816,
+      "loss": 0.8057,
+      "reward": 0.2708333395421505,
+      "reward_std": 0.3334706127643585,
+      "rewards/accuracy_reward": 0.2708333395421505,
       "step": 49
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 497.22918701171875,
+      "completion_length": 1194.7708740234375,
       "epoch": 0.05,
-      "grad_norm": 2.4330465227691427,
-      "kl": 0.0083160400390625,
+      "grad_norm": 34.00219523785431,
+      "kl": 6.484375,
       "learning_rate": 5e-07,
-      "loss": -0.0264,
-      "reward": 0.7083333432674408,
-      "reward_std": 0.19867133349180222,
-      "rewards/tag_count_reward": 0.7083333432674408,
+      "loss": 0.8266,
+      "reward": 0.4583333432674408,
+      "reward_std": 0.48533491790294647,
+      "rewards/accuracy_reward": 0.4583333432674408,
       "step": 50
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 439.3125,
+      "completion_length": 1239.1458740234375,
       "epoch": 0.051,
-      "grad_norm": 2.8050886122578973,
-      "kl": 0.0108795166015625,
+      "grad_norm": 26.69602841803911,
+      "kl": 6.390625,
       "learning_rate": 5.1e-07,
-      "loss": 0.0674,
-      "reward": 0.7968750298023224,
-      "reward_std": 0.22688449919223785,
-      "rewards/tag_count_reward": 0.7968750298023224,
+      "loss": 0.8394,
+      "reward": 0.3750000149011612,
+      "reward_std": 0.4620174169540405,
+      "rewards/accuracy_reward": 0.3750000149011612,
       "step": 51
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 581.4375305175781,
+      "completion_length": 1502.3333740234375,
       "epoch": 0.052,
-      "grad_norm": 2.64701027690505,
-      "kl": 0.008026123046875,
+      "grad_norm": 48.25943319293833,
+      "kl": 6.84375,
       "learning_rate": 5.2e-07,
-      "loss": 0.1935,
-      "reward": 0.8125000298023224,
-      "reward_std": 0.2059411108493805,
-      "rewards/tag_count_reward": 0.8125000298023224,
+      "loss": 0.5298,
+      "reward": 0.1041666679084301,
+      "reward_std": 0.26679350435733795,
+      "rewards/accuracy_reward": 0.1041666679084301,
       "step": 52
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 441.81251525878906,
+      "completion_length": 1149.8125610351562,
       "epoch": 0.053,
-      "grad_norm": 3.023119957597956,
-      "kl": 0.01275634765625,
+      "grad_norm": 10.859425077715859,
+      "kl": 4.21875,
       "learning_rate": 5.3e-07,
-      "loss": -0.0015,
-      "reward": 0.8072916865348816,
-      "reward_std": 0.2015344277024269,
-      "rewards/tag_count_reward": 0.8072916865348816,
+      "loss": 0.6236,
+      "reward": 0.3541666716337204,
+      "reward_std": 0.4527246206998825,
+      "rewards/accuracy_reward": 0.3541666716337204,
       "step": 53
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 440.87501525878906,
+      "completion_length": 1191.2917175292969,
       "epoch": 0.054,
-      "grad_norm": 2.901413813399593,
-      "kl": 0.01080322265625,
+      "grad_norm": 8.797140904639873,
+      "kl": 3.703125,
       "learning_rate": 5.4e-07,
-      "loss": 0.0923,
-      "reward": 0.8802083432674408,
-      "reward_std": 0.1856703907251358,
-      "rewards/tag_count_reward": 0.8802083432674408,
+      "loss": 0.5783,
+      "reward": 0.25,
+      "reward_std": 0.3647414445877075,
+      "rewards/accuracy_reward": 0.25,
       "step": 54
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 430.16668701171875,
+      "completion_length": 1368.5833740234375,
       "epoch": 0.055,
-      "grad_norm": 3.111751227197174,
-      "kl": 0.016357421875,
+      "grad_norm": 19.19125385206527,
+      "kl": 3.6796875,
       "learning_rate": 5.5e-07,
-      "loss": 0.1086,
-      "reward": 0.890625,
-      "reward_std": 0.18489186465740204,
-      "rewards/tag_count_reward": 0.890625,
+      "loss": 0.3792,
+      "reward": 0.14583333395421505,
+      "reward_std": 0.20272701978683472,
+      "rewards/accuracy_reward": 0.14583333395421505,
       "step": 55
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 427.3958435058594,
+      "completion_length": 1439.5416870117188,
       "epoch": 0.056,
-      "grad_norm": 2.8553539008783244,
-      "kl": 0.0155029296875,
+      "grad_norm": 12.93745800444744,
+      "kl": 3.09375,
       "learning_rate": 5.6e-07,
-      "loss": 0.1519,
-      "reward": 0.9166666865348816,
-      "reward_std": 0.1548745259642601,
-      "rewards/tag_count_reward": 0.9166666865348816,
+      "loss": 0.5505,
+      "reward": 0.2708333432674408,
+      "reward_std": 0.46229176223278046,
+      "rewards/accuracy_reward": 0.2708333432674408,
       "step": 56
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 437.8125,
+      "completion_length": 1434.1458740234375,
       "epoch": 0.057,
-      "grad_norm": 2.925778803233364,
-      "kl": 0.0120849609375,
+      "grad_norm": 6.314504912375571,
+      "kl": 2.7421875,
       "learning_rate": 5.699999999999999e-07,
-      "loss": 0.1911,
-      "reward": 0.9479166865348816,
-      "reward_std": 0.09902853146195412,
-      "rewards/tag_count_reward": 0.9479166865348816,
+      "loss": 0.3593,
+      "reward": 0.12500000558793545,
+      "reward_std": 0.26742906868457794,
+      "rewards/accuracy_reward": 0.12500000558793545,
       "step": 57
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 404.25,
+      "completion_length": 1560.8125610351562,
       "epoch": 0.058,
-      "grad_norm": 1.6735431769305273,
-      "kl": 0.0142822265625,
+      "grad_norm": 8.178175914094309,
+      "kl": 2.25,
       "learning_rate": 5.8e-07,
-      "loss": 0.0434,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.2542,
+      "reward": 0.08333333395421505,
+      "reward_std": 0.18523553758859634,
+      "rewards/accuracy_reward": 0.08333333395421505,
       "step": 58
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 412.9791717529297,
+      "completion_length": 1669.5000610351562,
       "epoch": 0.059,
-      "grad_norm": 1.7942238123951728,
-      "kl": 0.0159912109375,
+      "grad_norm": 21.034515284436043,
+      "kl": 2.0625,
       "learning_rate": 5.9e-07,
-      "loss": -0.0323,
-      "reward": 0.984375,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.1423,
+      "reward": 0.06250000186264515,
+      "reward_std": 0.16948115825653076,
+      "rewards/accuracy_reward": 0.06250000186264515,
       "step": 59
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 427.25001525878906,
+      "completion_length": 1772.854248046875,
       "epoch": 0.06,
-      "grad_norm": 2.3794770067519453,
-      "kl": 0.009979248046875,
+      "grad_norm": 17.559304505591005,
+      "kl": 1.29296875,
       "learning_rate": 6e-07,
-      "loss": 0.0452,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.060412485152482986,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0542,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 60
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 454.4583435058594,
+      "completion_length": 1812.6666870117188,
       "epoch": 0.061,
-      "grad_norm": 1.8579560006684275,
-      "kl": 0.0142822265625,
+      "grad_norm": 10.090220783344812,
+      "kl": 0.689453125,
       "learning_rate": 6.1e-07,
-      "loss": 0.0847,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.046308886259794235,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.1402,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.14433756470680237,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 61
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 400.6041717529297,
+      "completion_length": 1782.3541870117188,
       "epoch": 0.062,
-      "grad_norm": 0.10582058294459042,
-      "kl": 0.009918212890625,
+      "grad_norm": 4.247427187524467,
+      "kl": 0.5390625,
       "learning_rate": 6.2e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.082,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 62
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 397.8333435058594,
+      "completion_length": 1778.1250610351562,
       "epoch": 0.063,
-      "grad_norm": 2.117355134501401,
-      "kl": 0.0135498046875,
+      "grad_norm": 8.959241439282266,
+      "kl": 0.50390625,
       "learning_rate": 6.3e-07,
-      "loss": 0.0914,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0203,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 63
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 366.4791717529297,
+      "completion_length": 1899.625,
       "epoch": 0.064,
-      "grad_norm": 1.2394704841216948,
-      "kl": 0.009429931640625,
+      "grad_norm": 4.245054272954011,
+      "kl": 0.3955078125,
       "learning_rate": 6.4e-07,
-      "loss": 0.0065,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0773,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 64
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 396.7083435058594,
+      "completion_length": 1712.8958740234375,
       "epoch": 0.065,
-      "grad_norm": 0.1021771380380073,
-      "kl": 0.01220703125,
+      "grad_norm": 3.7901437352672955,
+      "kl": 0.400390625,
       "learning_rate": 6.5e-07,
-      "loss": 0.0005,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0754,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 65
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 395.8333435058594,
+      "completion_length": 1903.916748046875,
       "epoch": 0.066,
-      "grad_norm": 0.08440259324542373,
-      "kl": 0.0106201171875,
+      "grad_norm": 9.486455718480443,
+      "kl": 0.595703125,
       "learning_rate": 6.6e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
+      "loss": 0.0239,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 66
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 413.60418701171875,
+      "completion_length": 1879.75,
       "epoch": 0.067,
-      "grad_norm": 0.1441362554545231,
-      "kl": 0.014984130859375,
+      "grad_norm": 5.608530642989984,
+      "kl": 0.455078125,
       "learning_rate": 6.7e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0492,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.09731236100196838,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 67
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 350.4583435058594,
+      "completion_length": 1942.0625610351562,
       "epoch": 0.068,
-      "grad_norm": 0.11236301409587185,
-      "kl": 0.0130615234375,
+      "grad_norm": 2.9423044787381016,
+      "kl": 0.4697265625,
       "learning_rate": 6.800000000000001e-07,
-      "loss": 0.0005,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0826,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 68
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 414.4583435058594,
+      "completion_length": 1781.1666870117188,
       "epoch": 0.069,
-      "grad_norm": 0.09782498202426738,
-      "kl": 0.010986328125,
+      "grad_norm": 1.623547310774517,
+      "kl": 0.4541015625,
       "learning_rate": 6.9e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0687,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 69
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 418.4791717529297,
+      "completion_length": 1960.6666870117188,
       "epoch": 0.07,
-      "grad_norm": 0.09408510903169646,
-      "kl": 0.012115478515625,
+      "grad_norm": 2.2004686475174844,
+      "kl": 0.40625,
       "learning_rate": 7e-07,
-      "loss": 0.0005,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0707,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 70
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 349.5833435058594,
+      "completion_length": 1932.2500610351562,
       "epoch": 0.071,
-      "grad_norm": 0.1727501471584912,
-      "kl": 0.014862060546875,
+      "grad_norm": 7.538751131478555,
+      "kl": 0.546875,
       "learning_rate": 7.1e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
+      "loss": 0.0219,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 71
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 379.5208435058594,
+      "completion_length": 1901.0625610351562,
       "epoch": 0.072,
-      "grad_norm": 0.14195377794463795,
-      "kl": 0.012359619140625,
+      "grad_norm": 3.2022360945981743,
+      "kl": 0.484375,
       "learning_rate": 7.2e-07,
-      "loss": 0.0005,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0762,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 72
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 373.04168701171875,
+      "completion_length": 1870.0625610351562,
       "epoch": 0.073,
-      "grad_norm": 0.08441331456062105,
-      "kl": 0.011932373046875,
+      "grad_norm": 3.1779584422602585,
+      "kl": 0.466796875,
       "learning_rate": 7.3e-07,
-      "loss": 0.0005,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0689,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 73
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 414.5625,
+      "completion_length": 1910.3541870117188,
       "epoch": 0.074,
-      "grad_norm": 1.1861509967700965,
-      "kl": 0.010528564453125,
+      "grad_norm": 1.7827953190675647,
+      "kl": 0.607421875,
       "learning_rate": 7.4e-07,
-      "loss": -0.0065,
-      "reward": 0.984375,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0799,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 74
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 427.625,
+      "completion_length": 1991.2916870117188,
       "epoch": 0.075,
-      "grad_norm": 1.4796899488926614,
-      "kl": 0.013580322265625,
+      "grad_norm": 1.3662550722730615,
+      "kl": 0.58203125,
       "learning_rate": 7.5e-07,
-      "loss": 0.0271,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0832,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 75
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 371.50001525878906,
+      "completion_length": 1581.8125610351562,
       "epoch": 0.076,
-      "grad_norm": 0.09787357397649554,
-      "kl": 0.010894775390625,
+      "grad_norm": 26.20219244436764,
+      "kl": 0.66015625,
       "learning_rate": 7.599999999999999e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.2972,
+      "reward": 0.1458333358168602,
+      "reward_std": 0.2983023002743721,
+      "rewards/accuracy_reward": 0.1458333358168602,
       "step": 76
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 490.9583435058594,
+      "completion_length": 1759.3333740234375,
       "epoch": 0.077,
-      "grad_norm": 0.07637469397246856,
-      "kl": 0.0113525390625,
+      "grad_norm": 15.135976702568817,
+      "kl": 0.73828125,
       "learning_rate": 7.699999999999999e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.241,
+      "reward": 0.08333333395421505,
+      "reward_std": 0.24164992570877075,
+      "rewards/accuracy_reward": 0.08333333395421505,
       "step": 77
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 358.7708435058594,
+      "completion_length": 1619.7916870117188,
       "epoch": 0.078,
-      "grad_norm": 1.6056766807978617,
-      "kl": 0.010772705078125,
+      "grad_norm": 26.315773134879432,
+      "kl": 1.05078125,
       "learning_rate": 7.799999999999999e-07,
-      "loss": 0.0164,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.3486,
+      "reward": 0.1458333358168602,
+      "reward_std": 0.3547166883945465,
+      "rewards/accuracy_reward": 0.1458333358168602,
       "step": 78
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 367.625,
+      "completion_length": 1661.2291870117188,
       "epoch": 0.079,
-      "grad_norm": 2.052310159087112,
-      "kl": 0.0205078125,
+      "grad_norm": 26.632811476757965,
+      "kl": 1.31640625,
       "learning_rate": 7.9e-07,
-      "loss": -0.0571,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.3817,
+      "reward": 0.1666666716337204,
+      "reward_std": 0.3647414296865463,
+      "rewards/accuracy_reward": 0.1666666716337204,
       "step": 79
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 392.25,
+      "completion_length": 1795.1875,
       "epoch": 0.08,
-      "grad_norm": 1.7445790059715878,
-      "kl": 0.01043701171875,
+      "grad_norm": 11.265033528577549,
+      "kl": 1.953125,
       "learning_rate": 8e-07,
-      "loss": -0.0415,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.3093,
+      "reward": 0.0833333358168602,
+      "reward_std": 0.28867512941360474,
+      "rewards/accuracy_reward": 0.0833333358168602,
       "step": 80
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 403.1458435058594,
+      "completion_length": 1505.5,
       "epoch": 0.081,
-      "grad_norm": 0.08372535905276965,
-      "kl": 0.01226806640625,
+      "grad_norm": 15.482768455667852,
+      "kl": 1.875,
       "learning_rate": 8.1e-07,
-      "loss": 0.0005,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.3752,
+      "reward": 0.18750000558793545,
+      "reward_std": 0.3139677047729492,
+      "rewards/accuracy_reward": 0.18750000558793545,
       "step": 81
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 377.0208435058594,
+      "completion_length": 1413.9166870117188,
       "epoch": 0.082,
-      "grad_norm": 0.07293482460029307,
-      "kl": 0.0091552734375,
+      "grad_norm": 17.300650350853026,
+      "kl": 2.59375,
       "learning_rate": 8.199999999999999e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.535,
+      "reward": 0.2500000074505806,
+      "reward_std": 0.4314185529947281,
+      "rewards/accuracy_reward": 0.2500000074505806,
       "step": 82
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 376.7708435058594,
+      "completion_length": 1420.479248046875,
       "epoch": 0.083,
-      "grad_norm": 0.08117440047953302,
-      "kl": 0.010223388671875,
+      "grad_norm": 4.2202249221390495,
+      "kl": 3.6328125,
       "learning_rate": 8.299999999999999e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.5179,
+      "reward": 0.2708333432674408,
+      "reward_std": 0.33328525722026825,
+      "rewards/accuracy_reward": 0.2708333432674408,
       "step": 83
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 450.75001525878906,
+      "completion_length": 1532.9791870117188,
       "epoch": 0.084,
-      "grad_norm": 1.3785091767073692,
-      "kl": 0.01312255859375,
+      "grad_norm": 3.917613077952854,
+      "kl": 4.40625,
       "learning_rate": 8.399999999999999e-07,
-      "loss": 0.1838,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.6077,
+      "reward": 0.2500000149011612,
+      "reward_std": 0.436423659324646,
+      "rewards/accuracy_reward": 0.2500000149011612,
       "step": 84
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 378.93751525878906,
+      "completion_length": 1297.7708740234375,
       "epoch": 0.085,
-      "grad_norm": 0.08138495566859796,
-      "kl": 0.01055908203125,
+      "grad_norm": 5.151774404812395,
+      "kl": 4.5625,
       "learning_rate": 8.499999999999999e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.7407,
+      "reward": 0.3958333432674408,
+      "reward_std": 0.5092880129814148,
+      "rewards/accuracy_reward": 0.3958333432674408,
       "step": 85
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 414.0833435058594,
+      "completion_length": 1286.7291870117188,
       "epoch": 0.086,
-      "grad_norm": 0.07696676022736741,
-      "kl": 0.010498046875,
+      "grad_norm": 13.431136375347219,
+      "kl": 5.484375,
       "learning_rate": 8.599999999999999e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.7104,
+      "reward": 0.3125000149011612,
+      "reward_std": 0.44452594220638275,
+      "rewards/accuracy_reward": 0.3125000149011612,
       "step": 86
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 407.8958435058594,
+      "completion_length": 1359.3958740234375,
       "epoch": 0.087,
-      "grad_norm": 0.07467232706116313,
-      "kl": 0.009857177734375,
+      "grad_norm": 27.07087858046713,
+      "kl": 6.625,
       "learning_rate": 8.699999999999999e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.756,
+      "reward": 0.2708333432674408,
+      "reward_std": 0.3705311045050621,
+      "rewards/accuracy_reward": 0.2708333432674408,
       "step": 87
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 506.3333435058594,
+      "completion_length": 1545.4375610351562,
       "epoch": 0.088,
-      "grad_norm": 1.2191050233252494,
-      "kl": 0.013946533203125,
+      "grad_norm": 29.46777021366497,
+      "kl": 7.59375,
       "learning_rate": 8.799999999999999e-07,
-      "loss": 0.0024,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.7368,
+      "reward": 0.3125000149011612,
+      "reward_std": 0.4527246356010437,
+      "rewards/accuracy_reward": 0.3125000149011612,
       "step": 88
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 430.9583435058594,
+      "completion_length": 1183.5208435058594,
       "epoch": 0.089,
-      "grad_norm": 0.08723707900524888,
-      "kl": 0.010101318359375,
+      "grad_norm": 13.090984200418186,
+      "kl": 5.8125,
       "learning_rate": 8.9e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.8549,
+      "reward": 0.4166666865348816,
+      "reward_std": 0.49544864892959595,
+      "rewards/accuracy_reward": 0.4166666865348816,
       "step": 89
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 448.7083435058594,
+      "completion_length": 1142.1666870117188,
       "epoch": 0.09,
-      "grad_norm": 0.07242099149329641,
-      "kl": 0.01177978515625,
+      "grad_norm": 19.144646754285706,
+      "kl": 4.6875,
       "learning_rate": 9e-07,
-      "loss": 0.0005,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.578,
+      "reward": 0.2916666716337204,
+      "reward_std": 0.46793243288993835,
+      "rewards/accuracy_reward": 0.2916666716337204,
       "step": 90
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 474.00001525878906,
+      "completion_length": 1184.2708740234375,
       "epoch": 0.091,
-      "grad_norm": 0.06937379595279321,
-      "kl": 0.01031494140625,
+      "grad_norm": 8.905099769100909,
+      "kl": 4.140625,
       "learning_rate": 9.1e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.851,
+      "reward": 0.5,
+      "reward_std": 0.4872499704360962,
+      "rewards/accuracy_reward": 0.5,
       "step": 91
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 444.4583435058594,
+      "completion_length": 1181.2500610351562,
       "epoch": 0.092,
-      "grad_norm": 0.06533761500302811,
-      "kl": 0.0113525390625,
+      "grad_norm": 5.994460793321182,
+      "kl": 4.1484375,
       "learning_rate": 9.2e-07,
-      "loss": 0.0005,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.7321,
+      "reward": 0.3333333432674408,
+      "reward_std": 0.47405362129211426,
+      "rewards/accuracy_reward": 0.3333333432674408,
       "step": 92
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 429.7291717529297,
+      "completion_length": 1138.2291870117188,
       "epoch": 0.093,
-      "grad_norm": 0.06768204712004344,
-      "kl": 0.0106201171875,
+      "grad_norm": 3.9009203789046674,
+      "kl": 3.9609375,
       "learning_rate": 9.3e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.6356,
+      "reward": 0.3125,
+      "reward_std": 0.3823819011449814,
+      "rewards/accuracy_reward": 0.3125,
       "step": 93
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 488.1875305175781,
+      "completion_length": 1130.6042175292969,
       "epoch": 0.094,
-      "grad_norm": 0.07154676649029688,
-      "kl": 0.010498046875,
+      "grad_norm": 14.789274399412072,
+      "kl": 3.7421875,
       "learning_rate": 9.399999999999999e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.8314,
+      "reward": 0.5208333432674408,
+      "reward_std": 0.4936225861310959,
+      "rewards/accuracy_reward": 0.5208333432674408,
       "step": 94
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 453.62501525878906,
+      "completion_length": 1247.604248046875,
       "epoch": 0.095,
-      "grad_norm": 0.07456151743945785,
-      "kl": 0.011993408203125,
+      "grad_norm": 5.6969547809855525,
+      "kl": 4.296875,
       "learning_rate": 9.499999999999999e-07,
-      "loss": 0.0005,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.691,
+      "reward": 0.375,
+      "reward_std": 0.4665868580341339,
+      "rewards/accuracy_reward": 0.375,
       "step": 95
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 489.3958435058594,
+      "completion_length": 1207.979248046875,
       "epoch": 0.096,
-      "grad_norm": 0.066160922726964,
-      "kl": 0.011871337890625,
+      "grad_norm": 4.374652048218959,
+      "kl": 3.4609375,
       "learning_rate": 9.6e-07,
-      "loss": 0.0005,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.5901,
+      "reward": 0.4166666716337204,
+      "reward_std": 0.48533491790294647,
+      "rewards/accuracy_reward": 0.4166666716337204,
       "step": 96
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 474.37501525878906,
+      "completion_length": 1117.7917175292969,
       "epoch": 0.097,
-      "grad_norm": 0.06491250867856747,
-      "kl": 0.009765625,
+      "grad_norm": 15.032279892384746,
+      "kl": 3.453125,
       "learning_rate": 9.7e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.8255,
+      "reward": 0.5000000298023224,
+      "reward_std": 0.5111141204833984,
+      "rewards/accuracy_reward": 0.5000000298023224,
       "step": 97
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 428.375,
+      "completion_length": 1162.6666870117188,
       "epoch": 0.098,
-      "grad_norm": 0.06720556796337879,
-      "kl": 0.010955810546875,
+      "grad_norm": 5.1311358187488265,
+      "kl": 3.546875,
       "learning_rate": 9.8e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.6885,
+      "reward": 0.3750000149011612,
+      "reward_std": 0.3880225718021393,
+      "rewards/accuracy_reward": 0.3750000149011612,
       "step": 98
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 517.2708435058594,
+      "completion_length": 1395.5208740234375,
       "epoch": 0.099,
-      "grad_norm": 0.07566623409050405,
-      "kl": 0.0106201171875,
+      "grad_norm": 11.607589199075166,
+      "kl": 4.59375,
       "learning_rate": 9.9e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.5829,
+      "reward": 0.2708333432674408,
+      "reward_std": 0.3548656776547432,
+      "rewards/accuracy_reward": 0.2708333432674408,
       "step": 99
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 438.3541717529297,
+      "completion_length": 1237.8541870117188,
       "epoch": 0.1,
-      "grad_norm": 0.05388612872443928,
-      "kl": 0.010894775390625,
+      "grad_norm": 6.312793803526658,
+      "kl": 4.90625,
       "learning_rate": 1e-06,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.7798,
+      "reward": 0.4375000149011612,
+      "reward_std": 0.4936225861310959,
+      "rewards/accuracy_reward": 0.4375000149011612,
       "step": 100
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 486.4583435058594,
+      "completion_length": 1420.791748046875,
       "epoch": 0.101,
-      "grad_norm": 0.07044171391846912,
-      "kl": 0.0115966796875,
+      "grad_norm": 6.413314022448711,
+      "kl": 5.109375,
       "learning_rate": 9.999972584460056e-07,
-      "loss": 0.0005,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.6962,
+      "reward": 0.3541666716337204,
+      "reward_std": 0.4936225861310959,
+      "rewards/accuracy_reward": 0.3541666716337204,
       "step": 101
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 439.5833435058594,
+      "completion_length": 1355.8750610351562,
       "epoch": 0.102,
-      "grad_norm": 0.059048029504382746,
-      "kl": 0.00982666015625,
+      "grad_norm": 6.861762753706466,
+      "kl": 5.046875,
       "learning_rate": 9.999890338174275e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.7163,
+      "reward": 0.3750000149011612,
+      "reward_std": 0.48533496260643005,
+      "rewards/accuracy_reward": 0.3750000149011612,
       "step": 102
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 459.8333435058594,
+      "completion_length": 1246.1458740234375,
       "epoch": 0.103,
-      "grad_norm": 0.09184309136650912,
-      "kl": 0.01324462890625,
+      "grad_norm": 9.213245256723015,
+      "kl": 3.890625,
       "learning_rate": 9.999753262144804e-07,
-      "loss": 0.0005,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.7132,
+      "reward": 0.4375,
+      "reward_std": 0.5129401385784149,
+      "rewards/accuracy_reward": 0.4375,
       "step": 103
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 454.4583435058594,
+      "completion_length": 1521.8541870117188,
       "epoch": 0.104,
-      "grad_norm": 1.231549772721057,
-      "kl": 0.0106201171875,
+      "grad_norm": 7.7514326994085705,
+      "kl": 4.828125,
       "learning_rate": 9.999561358041868e-07,
-      "loss": 0.0137,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.6185,
+      "reward": 0.2708333358168602,
+      "reward_std": 0.42066924273967743,
+      "rewards/accuracy_reward": 0.2708333358168602,
       "step": 104
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 531.8541870117188,
+      "completion_length": 1847.854248046875,
       "epoch": 0.105,
-      "grad_norm": 0.04630714212471174,
-      "kl": 0.009552001953125,
+      "grad_norm": 27.894073636427674,
+      "kl": 5.703125,
       "learning_rate": 9.99931462820376e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.3883,
+      "reward": 0.1250000037252903,
+      "reward_std": 0.33896228671073914,
+      "rewards/accuracy_reward": 0.1250000037252903,
       "step": 105
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 469.12501525878906,
+      "completion_length": 1570.2083740234375,
       "epoch": 0.106,
-      "grad_norm": 0.040916036564967215,
-      "kl": 0.009063720703125,
+      "grad_norm": 8.744531040559671,
+      "kl": 4.140625,
       "learning_rate": 9.999013075636804e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.4939,
+      "reward": 0.2708333358168602,
+      "reward_std": 0.42886053025722504,
+      "rewards/accuracy_reward": 0.2708333358168602,
       "step": 106
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 449.50001525878906,
+      "completion_length": 1795.875,
       "epoch": 0.107,
-      "grad_norm": 0.03666755496102716,
-      "kl": 0.0082855224609375,
+      "grad_norm": 19.701752627633397,
+      "kl": 4.0234375,
       "learning_rate": 9.998656704015323e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.3043,
+      "reward": 0.2708333432674408,
+      "reward_std": 0.4565621018409729,
+      "rewards/accuracy_reward": 0.2708333432674408,
       "step": 107
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 497.33335876464844,
+      "completion_length": 1950.9583740234375,
       "epoch": 0.108,
-      "grad_norm": 0.06750587223762,
-      "kl": 0.00872802734375,
+      "grad_norm": 12.831142777670046,
+      "kl": 3.7890625,
       "learning_rate": 9.998245517681593e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.2301,
+      "reward": 0.1041666716337204,
+      "reward_std": 0.21037912368774414,
+      "rewards/accuracy_reward": 0.1041666716337204,
       "step": 108
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 466.93751525878906,
+      "completion_length": 1872.0000610351562,
       "epoch": 0.109,
-      "grad_norm": 0.07079864707100918,
-      "kl": 0.009246826171875,
+      "grad_norm": 11.772842416899998,
+      "kl": 3.1796875,
       "learning_rate": 9.997779521645791e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.2621,
+      "reward": 0.0625,
+      "reward_std": 0.16948114335536957,
+      "rewards/accuracy_reward": 0.0625,
       "step": 109
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 475.8958435058594,
+      "completion_length": 2025.1875,
       "epoch": 0.11,
-      "grad_norm": 1.0674719403023372,
-      "kl": 0.015533447265625,
+      "grad_norm": 17.092780812911244,
+      "kl": 2.546875,
       "learning_rate": 9.997258721585931e-07,
-      "loss": -0.0139,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.1004,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 110
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 440.1875,
+      "completion_length": 2048.0,
       "epoch": 0.111,
-      "grad_norm": 0.03989195456389211,
-      "kl": 0.009490966796875,
+      "grad_norm": 10.010837372488686,
+      "kl": 1.65625,
       "learning_rate": 9.996683123847795e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
+      "loss": 0.0662,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 111
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 442.0833435058594,
+      "completion_length": 2031.4791870117188,
       "epoch": 0.112,
-      "grad_norm": 0.04589218813730981,
-      "kl": 0.0080108642578125,
+      "grad_norm": 8.592699227270325,
+      "kl": 1.109375,
       "learning_rate": 9.996052735444862e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0702,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 112
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 529.1250152587891,
+      "completion_length": 2048.0,
       "epoch": 0.113,
-      "grad_norm": 0.054086545297594345,
-      "kl": 0.0091552734375,
+      "grad_norm": 3.2236732228527747,
+      "kl": 0.986328125,
       "learning_rate": 9.995367564058216e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
+      "loss": 0.0395,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 113
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 445.39585876464844,
+      "completion_length": 2048.0,
       "epoch": 0.114,
-      "grad_norm": 1.209655996583225,
-      "kl": 0.011749267578125,
+      "grad_norm": 1.2496619682636319,
+      "kl": 1.2109375,
       "learning_rate": 9.994627618036452e-07,
-      "loss": -0.002,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0485,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 114
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 452.95835876464844,
+      "completion_length": 2012.5625,
       "epoch": 0.115,
-      "grad_norm": 0.039887545798762906,
-      "kl": 0.0077362060546875,
+      "grad_norm": 0.7890902979604373,
+      "kl": 1.3125,
       "learning_rate": 9.993832906395582e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
+      "loss": 0.0535,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 115
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 491.8541717529297,
+      "completion_length": 2048.0,
       "epoch": 0.116,
-      "grad_norm": 0.05109503294914965,
-      "kl": 0.00970458984375,
+      "grad_norm": 1.3046245007692778,
+      "kl": 0.857421875,
       "learning_rate": 9.992983438818915e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
+      "loss": 0.0343,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 116
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 450.25,
+      "completion_length": 2048.0,
       "epoch": 0.117,
-      "grad_norm": 0.050184412277555976,
-      "kl": 0.0084075927734375,
+      "grad_norm": 2.7902722466301073,
+      "kl": 0.5439453125,
       "learning_rate": 9.992079225656944e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
+      "loss": 0.0217,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 117
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 464.25001525878906,
+      "completion_length": 2048.0,
       "epoch": 0.118,
-      "grad_norm": 0.04985779343703463,
-      "kl": 0.009490966796875,
+      "grad_norm": 1.54436925179059,
+      "kl": 0.333984375,
       "learning_rate": 9.991120277927223e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
+      "loss": 0.0134,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 118
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 461.81251525878906,
+      "completion_length": 2015.375,
       "epoch": 0.119,
-      "grad_norm": 0.07227541582028604,
-      "kl": 0.0108642578125,
+      "grad_norm": 1.4524790926901052,
+      "kl": 0.24609375,
       "learning_rate": 9.990106607314225e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
+      "loss": 0.0099,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 119
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 508.33335876464844,
+      "completion_length": 2048.0,
       "epoch": 0.12,
-      "grad_norm": 0.042175346801523864,
-      "kl": 0.00927734375,
+      "grad_norm": 1.109577324960446,
+      "kl": 0.21728515625,
       "learning_rate": 9.989038226169207e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
+      "loss": 0.0087,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 120
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 430.62501525878906,
+      "completion_length": 2048.0,
       "epoch": 0.121,
-      "grad_norm": 0.04214803283818787,
-      "kl": 0.008148193359375,
+      "grad_norm": 0.6736523786303484,
+      "kl": 0.21533203125,
       "learning_rate": 9.98791514751006e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
+      "loss": 0.0086,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 121
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 409.3333435058594,
+      "completion_length": 1969.2083740234375,
       "epoch": 0.122,
-      "grad_norm": 0.047501096439942436,
-      "kl": 0.008209228515625,
+      "grad_norm": 0.7678084023558526,
+      "kl": 0.14208984375,
       "learning_rate": 9.98673738502114e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
+      "loss": 0.0057,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 122
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 454.2708435058594,
+      "completion_length": 2048.0,
       "epoch": 0.123,
-      "grad_norm": 0.07310038780733595,
-      "kl": 0.0118408203125,
+      "grad_norm": 1.4218519150963098,
+      "kl": 0.302734375,
       "learning_rate": 9.985504953053113e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
+      "loss": 0.0121,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 123
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 458.8958435058594,
+      "completion_length": 2026.3541870117188,
       "epoch": 0.124,
-      "grad_norm": 0.07502052646036045,
-      "kl": 0.008758544921875,
+      "grad_norm": 1.0052649353842142,
+      "kl": 0.341796875,
       "learning_rate": 9.98421786662277e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
+      "loss": 0.0139,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 124
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 482.50001525878906,
+      "completion_length": 2048.0,
       "epoch": 0.125,
-      "grad_norm": 0.04546364063861285,
-      "kl": 0.0083770751953125,
+      "grad_norm": 0.6571753899517748,
+      "kl": 0.34375,
       "learning_rate": 9.982876141412855e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
+      "loss": 0.0138,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 125
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 442.9166717529297,
+      "completion_length": 2044.9375,
       "epoch": 0.126,
-      "grad_norm": 0.03944185524719935,
-      "kl": 0.007354736328125,
+      "grad_norm": 0.5440072217037435,
+      "kl": 0.18310546875,
       "learning_rate": 9.981479793771866e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
+      "loss": 0.0073,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 126
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 489.1458435058594,
+      "completion_length": 2011.4583740234375,
       "epoch": 0.127,
-      "grad_norm": 0.07225300807985488,
-      "kl": 0.013092041015625,
+      "grad_norm": 0.5008936079561156,
+      "kl": 0.154296875,
       "learning_rate": 9.98002884071386e-07,
-      "loss": 0.0005,
-      "reward": 1.0,
+      "loss": 0.0062,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 127
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 484.6875,
+      "completion_length": 2023.9791870117188,
       "epoch": 0.128,
-      "grad_norm": 0.11451764875833652,
-      "kl": 0.010528564453125,
+      "grad_norm": 0.551741935356332,
+      "kl": 0.124267578125,
       "learning_rate": 9.97852329991824e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
+      "loss": 0.005,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 128
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 476.3958435058594,
+      "completion_length": 2042.3541870117188,
       "epoch": 0.129,
-      "grad_norm": 0.039811807244108155,
-      "kl": 0.008453369140625,
+      "grad_norm": 3.3721347923564573,
+      "kl": 0.088134765625,
       "learning_rate": 9.976963189729547e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.012,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 129
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 446.3541717529297,
+      "completion_length": 2012.5208740234375,
       "epoch": 0.13,
-      "grad_norm": 0.5592846499570234,
-      "kl": 0.012420654296875,
+      "grad_norm": 8.460732257799167,
+      "kl": 0.080078125,
       "learning_rate": 9.975348529157229e-07,
-      "loss": 0.0005,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0583,
+      "reward": 0.06250000186264515,
+      "reward_std": 0.21650634706020355,
+      "rewards/accuracy_reward": 0.06250000186264515,
       "step": 130
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 517.3125,
+      "completion_length": 2001.2916870117188,
       "epoch": 0.131,
-      "grad_norm": 0.05179490828304956,
-      "kl": 0.0108642578125,
+      "grad_norm": 0.3470282664013102,
+      "kl": 0.079833984375,
       "learning_rate": 9.973679337875418e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
+      "loss": 0.0032,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 131
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 508.0416717529297,
+      "completion_length": 1997.0000610351562,
       "epoch": 0.132,
-      "grad_norm": 0.05367694843444582,
-      "kl": 0.00897216796875,
+      "grad_norm": 0.4974380217582725,
+      "kl": 0.07421875,
       "learning_rate": 9.971955636222684e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
+      "loss": 0.003,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 132
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 424.3333435058594,
+      "completion_length": 1910.8750610351562,
       "epoch": 0.133,
-      "grad_norm": 0.06917189855894977,
-      "kl": 0.010284423828125,
+      "grad_norm": 3.1415628983138317,
+      "kl": 0.0830078125,
       "learning_rate": 9.970177445201783e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0378,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 133
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 471.66668701171875,
+      "completion_length": 1949.2708740234375,
       "epoch": 0.134,
-      "grad_norm": 0.06043455789032839,
-      "kl": 0.0084075927734375,
+      "grad_norm": 0.6967169753565179,
+      "kl": 0.082275390625,
       "learning_rate": 9.968344786479415e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
+      "loss": 0.0033,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 134
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 399.43751525878906,
+      "completion_length": 1935.7708740234375,
       "epoch": 0.135,
-      "grad_norm": 0.03641136487333736,
-      "kl": 0.0073699951171875,
+      "grad_norm": 3.5096095619799965,
+      "kl": 0.082763671875,
       "learning_rate": 9.96645768238595e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0367,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.09731236100196838,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 135
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 451.12501525878906,
+      "completion_length": 1899.6041870117188,
       "epoch": 0.136,
-      "grad_norm": 1.112101379858347,
-      "kl": 0.0093841552734375,
+      "grad_norm": 9.485496751543996,
+      "kl": 0.093017578125,
       "learning_rate": 9.964516155915151e-07,
-      "loss": -0.007,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.1034,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.14433756470680237,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 136
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 529.8333435058594,
+      "completion_length": 1824.6458740234375,
       "epoch": 0.137,
-      "grad_norm": 0.03110825958701824,
-      "kl": 0.008819580078125,
+      "grad_norm": 12.632155707183774,
+      "kl": 0.12548828125,
       "learning_rate": 9.962520230723906e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.131,
+      "reward": 0.1666666716337204,
+      "reward_std": 0.3234458789229393,
+      "rewards/accuracy_reward": 0.1666666716337204,
       "step": 137
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 481.7708435058594,
+      "completion_length": 1719.4583740234375,
       "epoch": 0.138,
-      "grad_norm": 0.03882544215511118,
-      "kl": 0.009429931640625,
+      "grad_norm": 17.828740955102756,
+      "kl": 0.1796875,
       "learning_rate": 9.960469931131936e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.2101,
+      "reward": 0.3125,
+      "reward_std": 0.3592497631907463,
+      "rewards/accuracy_reward": 0.3125,
       "step": 138
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 446.60418701171875,
+      "completion_length": 1424.9791870117188,
       "epoch": 0.139,
-      "grad_norm": 0.03389657927366085,
-      "kl": 0.0071258544921875,
+      "grad_norm": 15.394545638428058,
+      "kl": 0.2744140625,
       "learning_rate": 9.958365282121496e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1976,
+      "reward": 0.2083333358168602,
+      "reward_std": 0.32399246096611023,
+      "rewards/accuracy_reward": 0.2083333358168602,
       "step": 139
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 458.6458435058594,
+      "completion_length": 1344.854248046875,
       "epoch": 0.14,
-      "grad_norm": 0.03482197048060803,
-      "kl": 0.0071563720703125,
+      "grad_norm": 7.221885701007461,
+      "kl": 0.41796875,
       "learning_rate": 9.956206309337066e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1222,
+      "reward": 0.1458333432674408,
+      "reward_std": 0.20272701978683472,
+      "rewards/accuracy_reward": 0.1458333432674408,
       "step": 140
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 509.25001525878906,
+      "completion_length": 1181.0416870117188,
       "epoch": 0.141,
-      "grad_norm": 0.033669621174171584,
-      "kl": 0.0080108642578125,
+      "grad_norm": 3.4881054015114676,
+      "kl": 0.58203125,
       "learning_rate": 9.953993039085048e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0745,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 141
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 473.3958435058594,
+      "completion_length": 1301.0416870117188,
       "epoch": 0.142,
-      "grad_norm": 1.3199912436596453,
-      "kl": 0.0085601806640625,
+      "grad_norm": 6.265444386601574,
+      "kl": 0.931640625,
       "learning_rate": 9.951725498333448e-07,
-      "loss": 0.001,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.1929,
+      "reward": 0.1666666679084301,
+      "reward_std": 0.2748958021402359,
+      "rewards/accuracy_reward": 0.1666666679084301,
       "step": 142
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 414.9375,
+      "completion_length": 862.7291870117188,
       "epoch": 0.143,
-      "grad_norm": 0.0685302014271952,
-      "kl": 0.010650634765625,
+      "grad_norm": 12.800812361593014,
+      "kl": 0.814453125,
       "learning_rate": 9.949403714711526e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.2035,
+      "reward": 0.2291666716337204,
+      "reward_std": 0.3703821003437042,
+      "rewards/accuracy_reward": 0.2291666716337204,
       "step": 143
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 442.37501525878906,
+      "completion_length": 773.3958740234375,
       "epoch": 0.144,
-      "grad_norm": 0.043254836515515906,
-      "kl": 0.0075225830078125,
+      "grad_norm": 8.775676662554263,
+      "kl": 1.1484375,
       "learning_rate": 9.947027716509488e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1703,
+      "reward": 0.2083333395421505,
+      "reward_std": 0.345238521695137,
+      "rewards/accuracy_reward": 0.2083333395421505,
       "step": 144
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 447.2708435058594,
+      "completion_length": 830.0833740234375,
       "epoch": 0.145,
-      "grad_norm": 0.034540457224639255,
-      "kl": 0.008514404296875,
+      "grad_norm": 7.779397255668793,
+      "kl": 1.484375,
       "learning_rate": 9.944597532678119e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.2228,
+      "reward": 0.27083333395421505,
+      "reward_std": 0.3296331316232681,
+      "rewards/accuracy_reward": 0.27083333395421505,
       "step": 145
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 488.9583435058594,
+      "completion_length": 745.1041870117188,
       "epoch": 0.146,
-      "grad_norm": 0.032769485521784394,
-      "kl": 0.008148193359375,
+      "grad_norm": 5.198810707205854,
+      "kl": 1.087890625,
       "learning_rate": 9.942113192828444e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1749,
+      "reward": 0.3541666865348816,
+      "reward_std": 0.4678434580564499,
+      "rewards/accuracy_reward": 0.3541666865348816,
       "step": 146
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 469.9375,
+      "completion_length": 759.7083435058594,
       "epoch": 0.147,
-      "grad_norm": 1.340172920356849,
-      "kl": 0.0115966796875,
+      "grad_norm": 8.851515015216153,
+      "kl": 1.1015625,
       "learning_rate": 9.939574727231362e-07,
-      "loss": 0.0266,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.3052,
+      "reward": 0.4791666716337204,
+      "reward_std": 0.4565621316432953,
+      "rewards/accuracy_reward": 0.4791666716337204,
       "step": 147
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 476.8333435058594,
+      "completion_length": 606.6666717529297,
       "epoch": 0.148,
-      "grad_norm": 0.033459278580343914,
-      "kl": 0.008331298828125,
+      "grad_norm": 8.177464887643172,
+      "kl": 0.7958984375,
       "learning_rate": 9.93698216681727e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0638,
+      "reward": 0.4791666716337204,
+      "reward_std": 0.498006671667099,
+      "rewards/accuracy_reward": 0.4791666716337204,
       "step": 148
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 456.75001525878906,
+      "completion_length": 584.9166717529297,
       "epoch": 0.149,
-      "grad_norm": 0.039450475594316296,
-      "kl": 0.0071868896484375,
+      "grad_norm": 7.656456457415272,
+      "kl": 0.30126953125,
       "learning_rate": 9.934335543175705e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1565,
+      "reward": 0.3750000149011612,
+      "reward_std": 0.49983277916908264,
+      "rewards/accuracy_reward": 0.3750000149011612,
       "step": 149
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 470.37501525878906,
+      "completion_length": 654.1666870117188,
       "epoch": 0.15,
-      "grad_norm": 0.031938722125296505,
-      "kl": 0.0070953369140625,
+      "grad_norm": 6.224798121709459,
+      "kl": 0.4150390625,
       "learning_rate": 9.931634888554935e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1107,
+      "reward": 0.4166666716337204,
+      "reward_std": 0.4622027724981308,
+      "rewards/accuracy_reward": 0.4166666716337204,
       "step": 150
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 430.47918701171875,
+      "completion_length": 734.1666870117188,
       "epoch": 0.151,
-      "grad_norm": 0.04371106287499349,
-      "kl": 0.0083160400390625,
+      "grad_norm": 7.274775043125759,
+      "kl": 1.57421875,
       "learning_rate": 9.928880235861588e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.442,
+      "reward": 0.6250000298023224,
+      "reward_std": 0.49983277916908264,
+      "rewards/accuracy_reward": 0.6250000298023224,
       "step": 151
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 426.2291717529297,
+      "completion_length": 703.2708435058594,
       "epoch": 0.152,
-      "grad_norm": 0.036907155124602,
-      "kl": 0.00762939453125,
+      "grad_norm": 6.864245556822268,
+      "kl": 0.98828125,
       "learning_rate": 9.926071618660237e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1851,
+      "reward": 0.5416666716337204,
+      "reward_std": 0.3492250144481659,
+      "rewards/accuracy_reward": 0.5416666716337204,
       "step": 152
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 481.1875,
+      "completion_length": 818.1250305175781,
       "epoch": 0.153,
-      "grad_norm": 0.04160779790370388,
-      "kl": 0.007904052734375,
+      "grad_norm": 11.408225853457964,
+      "kl": 2.21875,
       "learning_rate": 9.923209071172994e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.371,
+      "reward": 0.5625,
+      "reward_std": 0.5092880129814148,
+      "rewards/accuracy_reward": 0.5625,
       "step": 153
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 464.1875,
+      "completion_length": 777.0000305175781,
       "epoch": 0.154,
-      "grad_norm": 0.04461132836472903,
-      "kl": 0.00921630859375,
+      "grad_norm": 6.3358153764774485,
+      "kl": 0.5390625,
       "learning_rate": 9.9202926282791e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0957,
+      "reward": 0.7291666865348816,
+      "reward_std": 0.3898850083351135,
+      "rewards/accuracy_reward": 0.7291666865348816,
       "step": 154
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 495.375,
+      "completion_length": 919.7291870117188,
       "epoch": 0.155,
-      "grad_norm": 1.2860156647484209,
-      "kl": 0.007720947265625,
+      "grad_norm": 10.101212227331056,
+      "kl": 1.44140625,
       "learning_rate": 9.917322325514487e-07,
-      "loss": 0.0091,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.2374,
+      "reward": 0.5208333432674408,
+      "reward_std": 0.46229176223278046,
+      "rewards/accuracy_reward": 0.5208333432674408,
       "step": 155
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 394.7291717529297,
+      "completion_length": 1391.6041870117188,
       "epoch": 0.156,
-      "grad_norm": 0.04035318480378971,
-      "kl": 0.0077362060546875,
+      "grad_norm": 9.966611921979185,
+      "kl": 0.9140625,
       "learning_rate": 9.91429819907136e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1858,
+      "reward": 0.5416666716337204,
+      "reward_std": 0.45400412380695343,
+      "rewards/accuracy_reward": 0.5416666716337204,
       "step": 156
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 479.10418701171875,
+      "completion_length": 1638.5416870117188,
       "epoch": 0.157,
-      "grad_norm": 0.04387283645291692,
-      "kl": 0.0073699951171875,
+      "grad_norm": 8.0006546049797,
+      "kl": 0.783203125,
       "learning_rate": 9.911220285797748e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1432,
+      "reward": 0.4375000149011612,
+      "reward_std": 0.5010893195867538,
+      "rewards/accuracy_reward": 0.4375000149011612,
       "step": 157
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 448.9583435058594,
+      "completion_length": 1556.0833740234375,
       "epoch": 0.158,
-      "grad_norm": 0.038691858484494475,
-      "kl": 0.00836181640625,
+      "grad_norm": 5.562125684142098,
+      "kl": 1.26171875,
       "learning_rate": 9.908088623197048e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1654,
+      "reward": 0.4166666716337204,
+      "reward_std": 0.48533497750759125,
+      "rewards/accuracy_reward": 0.4166666716337204,
       "step": 158
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 479.6458435058594,
+      "completion_length": 1575.5833740234375,
       "epoch": 0.159,
-      "grad_norm": 0.03853129628943197,
-      "kl": 0.007293701171875,
+      "grad_norm": 5.92002687702095,
+      "kl": 0.966796875,
       "learning_rate": 9.904903249427582e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.066,
+      "reward": 0.5000000149011612,
+      "reward_std": 0.46966955065727234,
+      "rewards/accuracy_reward": 0.5000000149011612,
       "step": 159
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 406.56251525878906,
+      "completion_length": 1767.6666870117188,
       "epoch": 0.16,
-      "grad_norm": 1.1612045082634144,
-      "kl": 0.0088653564453125,
+      "grad_norm": 7.474059463824677,
+      "kl": 1.181640625,
       "learning_rate": 9.901664203302124e-07,
-      "loss": 0.0065,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.108,
+      "reward": 0.4791666865348816,
+      "reward_std": 0.5129401683807373,
+      "rewards/accuracy_reward": 0.4791666865348816,
       "step": 160
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 434.2291717529297,
+      "completion_length": 1830.9375610351562,
       "epoch": 0.161,
-      "grad_norm": 0.04436998873448987,
-      "kl": 0.0077362060546875,
+      "grad_norm": 12.534883001599582,
+      "kl": 0.5859375,
       "learning_rate": 9.89837152428743e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1223,
+      "reward": 0.3125000149011612,
+      "reward_std": 0.47795717418193817,
+      "rewards/accuracy_reward": 0.3125000149011612,
       "step": 161
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 447.8333435058594,
+      "completion_length": 1797.4375,
       "epoch": 0.162,
-      "grad_norm": 0.036244492000791544,
-      "kl": 0.0079345703125,
+      "grad_norm": 3.7574115477325427,
+      "kl": 0.728515625,
       "learning_rate": 9.895025252503755e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0551,
+      "reward": 0.2083333395421505,
+      "reward_std": 0.3391112834215164,
+      "rewards/accuracy_reward": 0.2083333395421505,
       "step": 162
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 458.4583435058594,
+      "completion_length": 1801.5625610351562,
       "epoch": 0.163,
-      "grad_norm": 0.04100771556606424,
-      "kl": 0.0080108642578125,
+      "grad_norm": 4.7156979351846,
+      "kl": 1.5078125,
       "learning_rate": 9.891625428724364e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1701,
+      "reward": 0.3125000149011612,
+      "reward_std": 0.46402889490127563,
+      "rewards/accuracy_reward": 0.3125000149011612,
       "step": 163
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 449.8333435058594,
+      "completion_length": 1687.0208740234375,
       "epoch": 0.164,
-      "grad_norm": 0.08359359800747004,
-      "kl": 0.0088653564453125,
+      "grad_norm": 8.06697964790504,
+      "kl": 0.9765625,
       "learning_rate": 9.888172094375033e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1436,
+      "reward": 0.4583333432674408,
+      "reward_std": 0.5185807943344116,
+      "rewards/accuracy_reward": 0.4583333432674408,
       "step": 164
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 468.2708435058594,
+      "completion_length": 1625.8541870117188,
       "epoch": 0.165,
-      "grad_norm": 0.04022627136874073,
-      "kl": 0.008453369140625,
+      "grad_norm": 4.60494506080584,
+      "kl": 1.1953125,
       "learning_rate": 9.88466529153356e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1505,
+      "reward": 0.3958333432674408,
+      "reward_std": 0.5010893195867538,
+      "rewards/accuracy_reward": 0.3958333432674408,
       "step": 165
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 459.87501525878906,
+      "completion_length": 1336.1666870117188,
       "epoch": 0.166,
-      "grad_norm": 0.04302669925116951,
-      "kl": 0.0085906982421875,
+      "grad_norm": 6.395172199078656,
+      "kl": 0.830078125,
       "learning_rate": 9.881105062929221e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0987,
+      "reward": 0.6041666716337204,
+      "reward_std": 0.39735177159309387,
+      "rewards/accuracy_reward": 0.6041666716337204,
       "step": 166
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 473.68751525878906,
+      "completion_length": 1312.8333740234375,
       "epoch": 0.167,
-      "grad_norm": 0.04557294629797261,
-      "kl": 0.008331298828125,
+      "grad_norm": 11.344241132693757,
+      "kl": 1.109375,
       "learning_rate": 9.877491451942284e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0342,
+      "reward": 0.6458333432674408,
+      "reward_std": 0.4796942472457886,
+      "rewards/accuracy_reward": 0.6458333432674408,
       "step": 167
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 442.5416717529297,
+      "completion_length": 1327.0833740234375,
       "epoch": 0.168,
-      "grad_norm": 0.03483320947401022,
-      "kl": 0.0066680908203125,
+      "grad_norm": 10.283790598970981,
+      "kl": 1.36328125,
       "learning_rate": 9.873824502603459e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.2304,
+      "reward": 0.7500000298023224,
+      "reward_std": 0.4314185529947281,
+      "rewards/accuracy_reward": 0.7500000298023224,
       "step": 168
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 493.35418701171875,
+      "completion_length": 1466.0208740234375,
       "epoch": 0.169,
-      "grad_norm": 1.2768670862840756,
-      "kl": 0.010406494140625,
+      "grad_norm": 7.047048175539603,
+      "kl": 1.388671875,
       "learning_rate": 9.870104259593362e-07,
-      "loss": 0.0031,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.1692,
+      "reward": 0.4791666865348816,
+      "reward_std": 0.36671652644872665,
+      "rewards/accuracy_reward": 0.4791666865348816,
       "step": 169
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 460.5208435058594,
+      "completion_length": 1697.3333740234375,
       "epoch": 0.17,
-      "grad_norm": 0.03919333998759371,
-      "kl": 0.0078582763671875,
+      "grad_norm": 5.516432594348777,
+      "kl": 1.044921875,
       "learning_rate": 9.866330768241983e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0891,
+      "reward": 0.5208333432674408,
+      "reward_std": 0.5167547464370728,
+      "rewards/accuracy_reward": 0.5208333432674408,
       "step": 170
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 444.7708435058594,
+      "completion_length": 1838.8750610351562,
       "epoch": 0.171,
-      "grad_norm": 0.04710061499517116,
-      "kl": 0.00848388671875,
+      "grad_norm": 9.116022478994891,
+      "kl": 0.79296875,
       "learning_rate": 9.862504074528126e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0876,
+      "reward": 0.2291666679084301,
+      "reward_std": 0.3961612284183502,
+      "rewards/accuracy_reward": 0.2291666679084301,
       "step": 171
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 470.62501525878906,
+      "completion_length": 1859.0833740234375,
       "epoch": 0.172,
-      "grad_norm": 0.03832441075014794,
-      "kl": 0.00909423828125,
+      "grad_norm": 13.26478515962402,
+      "kl": 0.859375,
       "learning_rate": 9.85862422507884e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.136,
+      "reward": 0.2083333432674408,
+      "reward_std": 0.4056393951177597,
+      "rewards/accuracy_reward": 0.2083333432674408,
       "step": 172
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 465.04168701171875,
+      "completion_length": 1942.0833740234375,
       "epoch": 0.173,
-      "grad_norm": 0.04435668106278205,
-      "kl": 0.00775146484375,
+      "grad_norm": 8.021047533882124,
+      "kl": 1.09765625,
       "learning_rate": 9.854691267168871e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0835,
+      "reward": 0.10416666977107525,
+      "reward_std": 0.2574043273925781,
+      "rewards/accuracy_reward": 0.10416666977107525,
       "step": 173
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 503.9791717529297,
+      "completion_length": 1823.1250610351562,
       "epoch": 0.174,
-      "grad_norm": 0.036439831353889465,
-      "kl": 0.00848388671875,
+      "grad_norm": 5.8895174740418605,
+      "kl": 0.78515625,
       "learning_rate": 9.850705248720068e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0685,
+      "reward": 0.4375000149011612,
+      "reward_std": 0.3334706127643585,
+      "rewards/accuracy_reward": 0.4375000149011612,
       "step": 174
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 481.6875,
+      "completion_length": 1656.0208740234375,
       "epoch": 0.175,
-      "grad_norm": 0.04549820373206854,
-      "kl": 0.008941650390625,
+      "grad_norm": 11.574044449276784,
+      "kl": 1.70703125,
       "learning_rate": 9.846666218300807e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0906,
+      "reward": 0.2083333432674408,
+      "reward_std": 0.22040385007858276,
+      "rewards/accuracy_reward": 0.2083333432674408,
       "step": 175
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 459.91668701171875,
+      "completion_length": 1427.6458740234375,
       "epoch": 0.176,
-      "grad_norm": 0.054447438694994736,
-      "kl": 0.0087890625,
+      "grad_norm": 7.497498321327927,
+      "kl": 1.54296875,
       "learning_rate": 9.8425742251254e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1637,
+      "reward": 0.2916666716337204,
+      "reward_std": 0.24164992570877075,
+      "rewards/accuracy_reward": 0.2916666716337204,
       "step": 176
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 488.2708435058594,
+      "completion_length": 1255.0208740234375,
       "epoch": 0.177,
-      "grad_norm": 0.04069149781366238,
-      "kl": 0.0084075927734375,
+      "grad_norm": 11.982508427718452,
+      "kl": 1.06640625,
       "learning_rate": 9.838429319053495e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.039,
+      "reward": 0.8333333432674408,
+      "reward_std": 0.12309150397777557,
+      "rewards/accuracy_reward": 0.8333333432674408,
       "step": 177
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 422.9583435058594,
+      "completion_length": 1285.979248046875,
       "epoch": 0.178,
-      "grad_norm": 0.05079796029083161,
-      "kl": 0.0077362060546875,
+      "grad_norm": 7.34303815424533,
+      "kl": 0.94921875,
       "learning_rate": 9.83423155058946e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1069,
+      "reward": 0.9375000298023224,
+      "reward_std": 0.21650634706020355,
+      "rewards/accuracy_reward": 0.9375000298023224,
       "step": 178
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 417.10418701171875,
+      "completion_length": 1326.0416870117188,
       "epoch": 0.179,
-      "grad_norm": 1.61418018880557,
-      "kl": 0.0090484619140625,
+      "grad_norm": 4.930972979153299,
+      "kl": 0.61328125,
       "learning_rate": 9.829980970881784e-07,
-      "loss": 0.0033,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0509,
+      "reward": 0.2291666716337204,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.2291666716337204,
       "step": 179
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 458.5625305175781,
+      "completion_length": 1417.541748046875,
       "epoch": 0.18,
-      "grad_norm": 1.1252202210444988,
-      "kl": 0.007904052734375,
+      "grad_norm": 9.097172655210604,
+      "kl": 0.50390625,
       "learning_rate": 9.825677631722435e-07,
-      "loss": -0.0051,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.1402,
+      "reward": 0.5416666865348816,
+      "reward_std": 0.3391112759709358,
+      "rewards/accuracy_reward": 0.5416666865348816,
       "step": 180
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 454.0208435058594,
+      "completion_length": 1601.125,
       "epoch": 0.181,
-      "grad_norm": 0.041347335947395474,
-      "kl": 0.0069580078125,
+      "grad_norm": 18.13863294242527,
+      "kl": 0.3642578125,
       "learning_rate": 9.821321585546243e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1801,
+      "reward": 0.5416666716337204,
+      "reward_std": 0.5036473274230957,
+      "rewards/accuracy_reward": 0.5416666716337204,
       "step": 181
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 514.6041870117188,
+      "completion_length": 1524.5625610351562,
       "epoch": 0.182,
-      "grad_norm": 1.1967680351799976,
-      "kl": 0.0073089599609375,
+      "grad_norm": 30.025939831028587,
+      "kl": 0.4609375,
       "learning_rate": 9.816912885430258e-07,
-      "loss": 0.0239,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.3146,
+      "reward": 0.4791666865348816,
+      "reward_std": 0.5129401683807373,
+      "rewards/accuracy_reward": 0.4791666865348816,
       "step": 182
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 490.0416717529297,
+      "completion_length": 1239.9583740234375,
       "epoch": 0.183,
-      "grad_norm": 0.038234135789257266,
-      "kl": 0.00762939453125,
+      "grad_norm": 13.021393015634885,
+      "kl": 0.376953125,
       "learning_rate": 9.812451585093098e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1619,
+      "reward": 0.8541666865348816,
+      "reward_std": 0.3395978510379791,
+      "rewards/accuracy_reward": 0.8541666865348816,
       "step": 183
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 500.31251525878906,
+      "completion_length": 952.9792175292969,
       "epoch": 0.184,
-      "grad_norm": 0.059067413317888416,
-      "kl": 0.008697509765625,
+      "grad_norm": 10.0017639721855,
+      "kl": 0.55078125,
       "learning_rate": 9.807937738894303e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1528,
+      "reward": 0.7500000298023224,
+      "reward_std": 0.24164992570877075,
+      "rewards/accuracy_reward": 0.7500000298023224,
       "step": 184
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 511.0625305175781,
+      "completion_length": 572.5208587646484,
       "epoch": 0.185,
-      "grad_norm": 0.05103624708512203,
-      "kl": 0.0093994140625,
+      "grad_norm": 5.1000511009894245,
+      "kl": 0.443359375,
       "learning_rate": 9.80337140183366e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0269,
+      "reward": 0.708333358168602,
+      "reward_std": 0.14433756470680237,
+      "rewards/accuracy_reward": 0.708333358168602,
       "step": 185
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 455.9791717529297,
+      "completion_length": 465.75001525878906,
       "epoch": 0.186,
-      "grad_norm": 0.05080899260139373,
-      "kl": 0.00958251953125,
+      "grad_norm": 10.308221109816788,
+      "kl": 0.39453125,
       "learning_rate": 9.798752629550546e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0301,
+      "reward": 0.5,
+      "reward_std": 0.14433756470680237,
+      "rewards/accuracy_reward": 0.5,
       "step": 186
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 483.0833435058594,
+      "completion_length": 496.54168701171875,
       "epoch": 0.187,
-      "grad_norm": 0.05430438471016196,
-      "kl": 0.00958251953125,
+      "grad_norm": 5.091587266137559,
+      "kl": 0.27685546875,
       "learning_rate": 9.794081478323245e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.015,
+      "reward": 0.75,
+      "reward_std": 0.22613351047039032,
+      "rewards/accuracy_reward": 0.75,
       "step": 187
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 433.7916717529297,
+      "completion_length": 506.35418701171875,
       "epoch": 0.188,
-      "grad_norm": 0.04681447901040309,
-      "kl": 0.0081634521484375,
+      "grad_norm": 4.180199549614713,
+      "kl": 0.27734375,
       "learning_rate": 9.78935800506826e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": -0.0327,
+      "reward": 0.5000000149011612,
+      "reward_std": 0.14433756470680237,
+      "rewards/accuracy_reward": 0.5000000149011612,
       "step": 188
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 458.9166717529297,
+      "completion_length": 627.0,
       "epoch": 0.189,
-      "grad_norm": 0.049365074467071016,
-      "kl": 0.010223388671875,
+      "grad_norm": 9.034678383867812,
+      "kl": 0.44140625,
       "learning_rate": 9.784582267339622e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": -0.0069,
+      "reward": 0.7291666865348816,
+      "reward_std": 0.21650634706020355,
+      "rewards/accuracy_reward": 0.7291666865348816,
       "step": 189
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 464.5833435058594,
+      "completion_length": 775.4375305175781,
       "epoch": 0.19,
-      "grad_norm": 0.047110399890504646,
-      "kl": 0.00836181640625,
+      "grad_norm": 7.821064988555408,
+      "kl": 0.4287109375,
       "learning_rate": 9.779754323328192e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1332,
+      "reward": 0.3333333432674408,
+      "reward_std": 0.20090095698833466,
+      "rewards/accuracy_reward": 0.3333333432674408,
       "step": 190
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 459.37501525878906,
+      "completion_length": 862.0,
       "epoch": 0.191,
-      "grad_norm": 0.05396861361472214,
-      "kl": 0.010223388671875,
+      "grad_norm": 6.629333161994175,
+      "kl": 0.4853515625,
       "learning_rate": 9.774874231860935e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0198,
+      "reward": 0.9375000298023224,
+      "reward_std": 0.16948114335536957,
+      "rewards/accuracy_reward": 0.9375000298023224,
       "step": 191
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 439.3333435058594,
+      "completion_length": 1099.3542175292969,
       "epoch": 0.192,
-      "grad_norm": 0.04360310974022942,
-      "kl": 0.009002685546875,
+      "grad_norm": 3.7474689455338863,
+      "kl": 0.482421875,
       "learning_rate": 9.769942052400235e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0713,
+      "reward": 0.2291666716337204,
+      "reward_std": 0.3592497631907463,
+      "rewards/accuracy_reward": 0.2291666716337204,
       "step": 192
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 463.0416717529297,
+      "completion_length": 1151.5416870117188,
       "epoch": 0.193,
-      "grad_norm": 0.0552391990077484,
-      "kl": 0.008544921875,
+      "grad_norm": 5.289444264237533,
+      "kl": 0.373046875,
       "learning_rate": 9.764957845043135e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1008,
+      "reward": 0.8125000298023224,
+      "reward_std": 0.3000393807888031,
+      "rewards/accuracy_reward": 0.8125000298023224,
       "step": 193
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 430.10418701171875,
+      "completion_length": 1005.0000305175781,
       "epoch": 0.194,
-      "grad_norm": 0.046843083270820006,
-      "kl": 0.00994873046875,
+      "grad_norm": 11.450710728556373,
+      "kl": 0.294921875,
       "learning_rate": 9.759921670520634e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1768,
+      "reward": 0.8125000298023224,
+      "reward_std": 0.4050038456916809,
+      "rewards/accuracy_reward": 0.8125000298023224,
       "step": 194
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 510.5416717529297,
+      "completion_length": 1228.8333740234375,
       "epoch": 0.195,
-      "grad_norm": 1.243856881724339,
-      "kl": 0.009368896484375,
+      "grad_norm": 5.264813218094335,
+      "kl": 0.4677734375,
       "learning_rate": 9.754833590196926e-07,
-      "loss": 0.0112,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.1286,
+      "reward": 0.6041666865348816,
+      "reward_std": 0.29257264733314514,
+      "rewards/accuracy_reward": 0.6041666865348816,
       "step": 195
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 499.25,
+      "completion_length": 1067.3750610351562,
       "epoch": 0.196,
-      "grad_norm": 0.04662672009974053,
-      "kl": 0.00970458984375,
+      "grad_norm": 5.193108908241919,
+      "kl": 0.365234375,
       "learning_rate": 9.749693666068663e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0447,
+      "reward": 0.875,
+      "reward_std": 0.28254789859056473,
+      "rewards/accuracy_reward": 0.875,
       "step": 196
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 453.81251525878906,
+      "completion_length": 1015.0416870117188,
       "epoch": 0.197,
-      "grad_norm": 0.04207520871736277,
-      "kl": 0.008514404296875,
+      "grad_norm": 8.155007374947248,
+      "kl": 0.38671875,
       "learning_rate": 9.744501960764203e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0391,
+      "reward": 0.8958333730697632,
+      "reward_std": 0.19526028633117676,
+      "rewards/accuracy_reward": 0.8958333730697632,
       "step": 197
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 444.8333435058594,
+      "completion_length": 1114.0416870117188,
       "epoch": 0.198,
-      "grad_norm": 1.2748791825422368,
-      "kl": 0.009185791015625,
+      "grad_norm": 8.86647781326395,
+      "kl": 0.52734375,
       "learning_rate": 9.739258537542835e-07,
-      "loss": -0.0019,
-      "reward": 0.984375,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0255,
+      "reward": 0.4583333544433117,
+      "reward_std": 0.2825479060411453,
+      "rewards/accuracy_reward": 0.4583333544433117,
       "step": 198
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 467.1458435058594,
+      "completion_length": 1067.2083435058594,
       "epoch": 0.199,
-      "grad_norm": 1.1242637108425146,
-      "kl": 0.010345458984375,
+      "grad_norm": 2.919664081055965,
+      "kl": 0.41015625,
       "learning_rate": 9.733963460294015e-07,
-      "loss": 0.0021,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0464,
+      "reward": 0.6458333432674408,
+      "reward_std": 0.21037912368774414,
+      "rewards/accuracy_reward": 0.6458333432674408,
       "step": 199
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 500.4791717529297,
+      "completion_length": 1127.4166870117188,
       "epoch": 0.2,
-      "grad_norm": 0.0364220480448336,
-      "kl": 0.0086669921875,
+      "grad_norm": 6.2005931833588,
+      "kl": 0.419921875,
       "learning_rate": 9.728616793536587e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0373,
+      "reward": 0.6458333432674408,
+      "reward_std": 0.4565621018409729,
+      "rewards/accuracy_reward": 0.6458333432674408,
       "step": 200
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 455.1666717529297,
+      "completion_length": 1100.5,
       "epoch": 0.201,
-      "grad_norm": 0.046692387506755174,
-      "kl": 0.008758544921875,
+      "grad_norm": 6.312579121540308,
+      "kl": 0.32275390625,
       "learning_rate": 9.723218602418e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0052,
+      "reward": 0.6250000298023224,
+      "reward_std": 0.20090095698833466,
+      "rewards/accuracy_reward": 0.6250000298023224,
       "step": 201
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 443.0833435058594,
+      "completion_length": 1308.8125610351562,
       "epoch": 0.202,
-      "grad_norm": 0.06278266766733745,
-      "kl": 0.0086669921875,
+      "grad_norm": 6.71934366386956,
+      "kl": 0.390625,
       "learning_rate": 9.717768952713511e-07,
-      "loss": 0.0003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0375,
+      "reward": 0.6666666865348816,
+      "reward_std": 0.36474141478538513,
+      "rewards/accuracy_reward": 0.6666666865348816,
       "step": 202
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 445.8125,
+      "completion_length": 1501.729248046875,
       "epoch": 0.203,
-      "grad_norm": 0.733144439217337,
-      "kl": 0.026031494140625,
+      "grad_norm": 8.205150600749949,
+      "kl": 0.443359375,
       "learning_rate": 9.71226791082538e-07,
-      "loss": 0.001,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0749,
+      "reward": 0.6041666865348816,
+      "reward_std": 0.42886053025722504,
+      "rewards/accuracy_reward": 0.6041666865348816,
       "step": 203
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 418.50001525878906,
+      "completion_length": 1443.1250610351562,
       "epoch": 0.204,
-      "grad_norm": 0.04988942656348295,
-      "kl": 0.0091552734375,
+      "grad_norm": 7.279568724256672,
+      "kl": 0.3232421875,
       "learning_rate": 9.706715543782064e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": -0.0261,
+      "reward": 0.7083333730697632,
+      "reward_std": 0.3805558532476425,
+      "rewards/accuracy_reward": 0.7083333730697632,
       "step": 204
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 448.5,
+      "completion_length": 1657.9166870117188,
       "epoch": 0.205,
-      "grad_norm": 1.4429664723338587,
-      "kl": 0.010711669921875,
+      "grad_norm": 5.664998276858177,
+      "kl": 0.2783203125,
       "learning_rate": 9.701111919237408e-07,
-      "loss": 0.0029,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0405,
+      "reward": 0.5416666716337204,
+      "reward_std": 0.45455068349838257,
+      "rewards/accuracy_reward": 0.5416666716337204,
       "step": 205
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 498.68751525878906,
+      "completion_length": 1765.1250610351562,
       "epoch": 0.206,
-      "grad_norm": 1.051585226601382,
-      "kl": 0.0103759765625,
+      "grad_norm": 5.90319144826754,
+      "kl": 0.39453125,
       "learning_rate": 9.695457105469804e-07,
-      "loss": -0.0194,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0529,
+      "reward": 0.3958333432674408,
+      "reward_std": 0.5010893195867538,
+      "rewards/accuracy_reward": 0.3958333432674408,
       "step": 206
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 510.8541717529297,
+      "completion_length": 1637.7083740234375,
       "epoch": 0.207,
-      "grad_norm": 0.051580365905477286,
-      "kl": 0.0103759765625,
+      "grad_norm": 3.960170295072406,
+      "kl": 0.24755859375,
       "learning_rate": 9.689751171381377e-07,
-      "loss": 0.0004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0137,
+      "reward": 0.5000000149011612,
+      "reward_std": 0.487981915473938,
+      "rewards/accuracy_reward": 0.5000000149011612,
       "step": 207
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 475.72918701171875,
+      "completion_length": 1554.3125610351562,
       "epoch": 0.208,
-      "grad_norm": 0.1551803755199804,
-      "kl": 0.014739990234375,
+      "grad_norm": 10.466240887383687,
+      "kl": 0.4130859375,
       "learning_rate": 9.683994186497132e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0854,
+      "reward": 0.4583333358168602,
+      "reward_std": 0.34349535405635834,
+      "rewards/accuracy_reward": 0.4583333358168602,
       "step": 208
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 512.9583435058594,
+      "completion_length": 1482.791748046875,
       "epoch": 0.209,
-      "grad_norm": 0.05272348357338901,
-      "kl": 0.012725830078125,
+      "grad_norm": 5.65807867444827,
+      "kl": 0.380859375,
       "learning_rate": 9.67818622096411e-07,
-      "loss": 0.0005,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0107,
+      "reward": 0.5000000149011612,
+      "reward_std": 0.46966952085494995,
+      "rewards/accuracy_reward": 0.5000000149011612,
       "step": 209
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 519.5833435058594,
+      "completion_length": 1453.0625610351562,
       "epoch": 0.21,
-      "grad_norm": 1.10417006874386,
-      "kl": 0.011077880859375,
+      "grad_norm": 4.633334454024917,
+      "kl": 0.609375,
       "learning_rate": 9.672327345550543e-07,
-      "loss": -0.0096,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.1067,
+      "reward": 0.5000000298023224,
+      "reward_std": 0.31771621108055115,
+      "rewards/accuracy_reward": 0.5000000298023224,
       "step": 210
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 530.3958435058594,
+      "completion_length": 1165.2708740234375,
       "epoch": 0.211,
-      "grad_norm": 1.1753773142145971,
-      "kl": 0.013519287109375,
+      "grad_norm": 6.728667133870936,
+      "kl": 0.576171875,
       "learning_rate": 9.666417631644976e-07,
-      "loss": 0.0078,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.1129,
+      "reward": 0.5208333432674408,
+      "reward_std": 0.4307829737663269,
+      "rewards/accuracy_reward": 0.5208333432674408,
       "step": 211
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 488.3125,
+      "completion_length": 1061.75,
       "epoch": 0.212,
-      "grad_norm": 1.4918163438086747,
-      "kl": 0.02587890625,
+      "grad_norm": 25.803415326699096,
+      "kl": 1.890625,
       "learning_rate": 9.66045715125541e-07,
-      "loss": 0.0116,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.04865618050098419,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.1127,
+      "reward": 0.4583333432674408,
+      "reward_std": 0.3295731097459793,
+      "rewards/accuracy_reward": 0.4583333432674408,
       "step": 212
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 487.4166717529297,
+      "completion_length": 1154.0833740234375,
       "epoch": 0.213,
-      "grad_norm": 2.1340988492664605,
-      "kl": 0.014801025390625,
+      "grad_norm": 13.976997329783389,
+      "kl": 1.48828125,
       "learning_rate": 9.654445977008414e-07,
-      "loss": 0.0412,
-      "reward": 0.984375,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0444,
+      "reward": 0.375,
+      "reward_std": 0.22040385007858276,
+      "rewards/accuracy_reward": 0.375,
       "step": 213
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 526.3958435058594,
+      "completion_length": 1197.2708740234375,
       "epoch": 0.214,
-      "grad_norm": 0.11688279967288762,
-      "kl": 0.015472412109375,
+      "grad_norm": 26.519196733572873,
+      "kl": 2.1640625,
       "learning_rate": 9.648384182148252e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0686,
+      "reward": 0.3541666865348816,
+      "reward_std": 0.3547166883945465,
+      "rewards/accuracy_reward": 0.3541666865348816,
       "step": 214
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 478.5416717529297,
+      "completion_length": 1264.9166870117188,
       "epoch": 0.215,
-      "grad_norm": 1.5894295793015976,
-      "kl": 0.017242431640625,
+      "grad_norm": 10.548359831171316,
+      "kl": 1.79296875,
       "learning_rate": 9.64227184053598e-07,
-      "loss": 0.0446,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.1041,
+      "reward": 0.5208333432674408,
+      "reward_std": 0.4565621316432953,
+      "rewards/accuracy_reward": 0.5208333432674408,
       "step": 215
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 468.02085876464844,
+      "completion_length": 1302.3958740234375,
       "epoch": 0.216,
-      "grad_norm": 0.08453494009874704,
-      "kl": 0.013946533203125,
+      "grad_norm": 9.542495479125368,
+      "kl": 0.951171875,
       "learning_rate": 9.636109026648554e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": -0.0425,
+      "reward": 0.2708333432674408,
+      "reward_std": 0.36671648919582367,
+      "rewards/accuracy_reward": 0.2708333432674408,
       "step": 216
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 545.625,
+      "completion_length": 1269.2291870117188,
       "epoch": 0.217,
-      "grad_norm": 0.07300918934288783,
-      "kl": 0.01556396484375,
+      "grad_norm": 17.069100759155155,
+      "kl": 0.98046875,
       "learning_rate": 9.629895815577915e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1957,
+      "reward": 0.6041666716337204,
+      "reward_std": 0.43651263415813446,
+      "rewards/accuracy_reward": 0.6041666716337204,
       "step": 217
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 423.9583435058594,
+      "completion_length": 1589.8333740234375,
       "epoch": 0.218,
-      "grad_norm": 0.0844738916631422,
-      "kl": 0.0145263671875,
+      "grad_norm": 12.947782548932052,
+      "kl": 1.171875,
       "learning_rate": 9.623632283030077e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1542,
+      "reward": 0.3125,
+      "reward_std": 0.25182366371154785,
+      "rewards/accuracy_reward": 0.3125,
       "step": 218
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 393.9583435058594,
+      "completion_length": 1564.4166870117188,
       "epoch": 0.219,
-      "grad_norm": 0.09820466302062966,
-      "kl": 0.01605224609375,
+      "grad_norm": 3.5153029336189436,
+      "kl": 0.71875,
       "learning_rate": 9.617318505324212e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0864,
+      "reward": 0.1041666679084301,
+      "reward_std": 0.26679350435733795,
+      "rewards/accuracy_reward": 0.1041666679084301,
       "step": 219
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 506.56251525878906,
+      "completion_length": 1616.0625,
       "epoch": 0.22,
-      "grad_norm": 1.1719180400121392,
-      "kl": 0.01727294921875,
+      "grad_norm": 2.1605123157631265,
+      "kl": 0.67578125,
       "learning_rate": 9.610954559391704e-07,
-      "loss": -0.0179,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0832,
+      "reward": 0.0625,
+      "reward_std": 0.11306675523519516,
+      "rewards/accuracy_reward": 0.0625,
       "step": 220
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 437.7083435058594,
+      "completion_length": 1817.1458740234375,
       "epoch": 0.221,
-      "grad_norm": 1.2988094815882822,
-      "kl": 0.016204833984375,
+      "grad_norm": 6.596542641335066,
+      "kl": 0.927734375,
       "learning_rate": 9.604540522775227e-07,
-      "loss": -0.0138,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.1033,
+      "reward": 0.1458333395421505,
+      "reward_std": 0.22604453563690186,
+      "rewards/accuracy_reward": 0.1458333395421505,
       "step": 221
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 475.9791717529297,
+      "completion_length": 1796.7916870117188,
       "epoch": 0.222,
-      "grad_norm": 1.2837249478628319,
-      "kl": 0.016510009765625,
+      "grad_norm": 9.102266381370802,
+      "kl": 1.02734375,
       "learning_rate": 9.598076473627796e-07,
-      "loss": 0.0012,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0911,
+      "reward": 0.2083333358168602,
+      "reward_std": 0.3861365020275116,
+      "rewards/accuracy_reward": 0.2083333358168602,
       "step": 222
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 450.7708435058594,
+      "completion_length": 1964.0833740234375,
       "epoch": 0.223,
-      "grad_norm": 1.253685798628967,
-      "kl": 0.016815185546875,
+      "grad_norm": 1.2829648988879188,
+      "kl": 0.84375,
       "learning_rate": 9.59156249071181e-07,
-      "loss": -0.003,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0506,
+      "reward": 0.06250000186264515,
+      "reward_std": 0.16948114335536957,
+      "rewards/accuracy_reward": 0.06250000186264515,
       "step": 223
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 543.1666717529297,
+      "completion_length": 1967.7500610351562,
       "epoch": 0.224,
-      "grad_norm": 1.30016273991412,
-      "kl": 0.01727294921875,
+      "grad_norm": 1.5300398116461542,
+      "kl": 0.83984375,
       "learning_rate": 9.58499865339809e-07,
-      "loss": 0.0038,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0577,
+      "reward": 0.0625,
+      "reward_std": 0.16948114335536957,
+      "rewards/accuracy_reward": 0.0625,
       "step": 224
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 434.12501525878906,
+      "completion_length": 2009.9583740234375,
       "epoch": 0.225,
-      "grad_norm": 0.09757944338988957,
-      "kl": 0.015350341796875,
+      "grad_norm": 3.080233390596236,
+      "kl": 0.783203125,
       "learning_rate": 9.578385041664925e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0425,
+      "reward": 0.06250000186264515,
+      "reward_std": 0.16948114335536957,
+      "rewards/accuracy_reward": 0.06250000186264515,
       "step": 225
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 518.5416870117188,
+      "completion_length": 1989.2916870117188,
       "epoch": 0.226,
-      "grad_norm": 1.2273247367534148,
-      "kl": 0.016143798828125,
+      "grad_norm": 3.8618696778328743,
+      "kl": 0.650390625,
       "learning_rate": 9.571721736097088e-07,
-      "loss": -0.0008,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0447,
+      "reward": 0.1875,
+      "reward_std": 0.2436249852180481,
+      "rewards/accuracy_reward": 0.1875,
       "step": 226
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 490.10418701171875,
+      "completion_length": 1952.0833740234375,
       "epoch": 0.227,
-      "grad_norm": 0.07686121660930782,
-      "kl": 0.01434326171875,
+      "grad_norm": 4.481020484432634,
+      "kl": 0.626953125,
       "learning_rate": 9.565008817884854e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0435,
+      "reward": 0.2500000074505806,
+      "reward_std": 0.3648904263973236,
+      "rewards/accuracy_reward": 0.2500000074505806,
       "step": 227
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 473.2708435058594,
+      "completion_length": 1854.8958740234375,
       "epoch": 0.228,
-      "grad_norm": 1.5759387434813044,
-      "kl": 0.014068603515625,
+      "grad_norm": 5.507016505385286,
+      "kl": 0.5390625,
       "learning_rate": 9.55824636882301e-07,
-      "loss": -0.004,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0383,
+      "reward": 0.2708333432674408,
+      "reward_std": 0.4370592087507248,
+      "rewards/accuracy_reward": 0.2708333432674408,
       "step": 228
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 471.50001525878906,
+      "completion_length": 1739.0625610351562,
       "epoch": 0.229,
-      "grad_norm": 0.21396356843199574,
-      "kl": 0.02099609375,
+      "grad_norm": 9.051605287585637,
+      "kl": 0.5078125,
       "learning_rate": 9.55143447130987e-07,
-      "loss": 0.0008,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0139,
+      "reward": 0.4166666865348816,
+      "reward_std": 0.3880225867033005,
+      "rewards/accuracy_reward": 0.4166666865348816,
       "step": 229
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 479.2083435058594,
+      "completion_length": 1741.2291870117188,
       "epoch": 0.23,
-      "grad_norm": 0.11709533688651282,
-      "kl": 0.0179443359375,
+      "grad_norm": 13.135796676279062,
+      "kl": 0.515625,
       "learning_rate": 9.54457320834625e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0845,
+      "reward": 0.7083333432674408,
+      "reward_std": 0.4622028023004532,
+      "rewards/accuracy_reward": 0.7083333432674408,
       "step": 230
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 478.8541717529297,
+      "completion_length": 1631.0625,
       "epoch": 0.231,
-      "grad_norm": 0.10027144130859894,
-      "kl": 0.018463134765625,
+      "grad_norm": 10.587323091599526,
+      "kl": 0.4033203125,
       "learning_rate": 9.537662663534477e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": -0.017,
+      "reward": 0.4375000149011612,
+      "reward_std": 0.4370592087507248,
+      "rewards/accuracy_reward": 0.4375000149011612,
       "step": 231
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 479.9583435058594,
+      "completion_length": 1668.1458740234375,
       "epoch": 0.232,
-      "grad_norm": 0.09671371423542396,
-      "kl": 0.01788330078125,
+      "grad_norm": 14.236060845278523,
+      "kl": 0.3681640625,
       "learning_rate": 9.530702921077358e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": -0.0284,
+      "reward": 0.6041666865348816,
+      "reward_std": 0.48542390763759613,
+      "rewards/accuracy_reward": 0.6041666865348816,
       "step": 232
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 491.8958435058594,
+      "completion_length": 1579.0000610351562,
       "epoch": 0.233,
-      "grad_norm": 0.14545470402187322,
-      "kl": 0.01849365234375,
+      "grad_norm": 19.83136815061087,
+      "kl": 0.439453125,
       "learning_rate": 9.523694065777156e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1491,
+      "reward": 0.458333358168602,
+      "reward_std": 0.4815203845500946,
+      "rewards/accuracy_reward": 0.458333358168602,
       "step": 233
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 496.72918701171875,
+      "completion_length": 1648.125,
       "epoch": 0.234,
-      "grad_norm": 1.198899919328826,
-      "kl": 0.0164794921875,
+      "grad_norm": 8.129358569499226,
+      "kl": 0.4130859375,
       "learning_rate": 9.516636183034564e-07,
-      "loss": 0.0263,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0552,
+      "reward": 0.7083333432674408,
+      "reward_std": 0.4622027724981308,
+      "rewards/accuracy_reward": 0.7083333432674408,
       "step": 234
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 433.12501525878906,
+      "completion_length": 1373.6041870117188,
       "epoch": 0.235,
-      "grad_norm": 0.09162266424217855,
-      "kl": 0.01885986328125,
+      "grad_norm": 18.553839384631928,
+      "kl": 0.4052734375,
       "learning_rate": 9.509529358847654e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0346,
+      "reward": 0.6666666865348816,
+      "reward_std": 0.462202787399292,
+      "rewards/accuracy_reward": 0.6666666865348816,
       "step": 235
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 595.0208435058594,
+      "completion_length": 1271.7291870117188,
       "epoch": 0.236,
-      "grad_norm": 1.1282448798264841,
-      "kl": 0.01708984375,
+      "grad_norm": 16.336867192574257,
+      "kl": 0.46484375,
       "learning_rate": 9.502373679810839e-07,
-      "loss": 0.1368,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.06226281076669693,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": -0.0754,
+      "reward": 0.3958333432674408,
+      "reward_std": 0.46019135415554047,
+      "rewards/accuracy_reward": 0.3958333432674408,
       "step": 236
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 524.6666870117188,
+      "completion_length": 1203.7291870117188,
       "epoch": 0.237,
-      "grad_norm": 0.10066116337470642,
-      "kl": 0.017547607421875,
+      "grad_norm": 19.410781464732917,
+      "kl": 0.439453125,
       "learning_rate": 9.495169233113806e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": -0.0556,
+      "reward": 0.458333358168602,
+      "reward_std": 0.49544869363307953,
+      "rewards/accuracy_reward": 0.458333358168602,
       "step": 237
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 565.1458435058594,
+      "completion_length": 1103.604248046875,
       "epoch": 0.238,
-      "grad_norm": 0.08271308069169885,
-      "kl": 0.0169677734375,
+      "grad_norm": 17.6092514611112,
+      "kl": 0.515625,
       "learning_rate": 9.487916106540465e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": -0.0954,
+      "reward": 0.3958333432674408,
+      "reward_std": 0.501089334487915,
+      "rewards/accuracy_reward": 0.3958333432674408,
       "step": 238
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 499.3125,
+      "completion_length": 1203.4375610351562,
       "epoch": 0.239,
-      "grad_norm": 0.0834749774046353,
-      "kl": 0.017578125,
+      "grad_norm": 11.060130216873306,
+      "kl": 0.904296875,
       "learning_rate": 9.480614388467877e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0811,
+      "reward": 0.520833358168602,
+      "reward_std": 0.5129401683807373,
+      "rewards/accuracy_reward": 0.520833358168602,
       "step": 239
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 516.7083435058594,
+      "completion_length": 1105.7291870117188,
       "epoch": 0.24,
-      "grad_norm": 1.082176127520902,
-      "kl": 0.016082763671875,
+      "grad_norm": 26.07429531999135,
+      "kl": 0.671875,
       "learning_rate": 9.473264167865171e-07,
-      "loss": 0.0141,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": -0.0323,
+      "reward": 0.3750000149011612,
+      "reward_std": 0.29821330308914185,
+      "rewards/accuracy_reward": 0.3750000149011612,
       "step": 240
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 512.125,
+      "completion_length": 1399.1458740234375,
       "epoch": 0.241,
-      "grad_norm": 0.07003007584923018,
-      "kl": 0.016632080078125,
+      "grad_norm": 54.01772747359288,
+      "kl": 0.970703125,
       "learning_rate": 9.465865534292464e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.234,
+      "reward": 0.1458333358168602,
+      "reward_std": 0.36410586535930634,
+      "rewards/accuracy_reward": 0.1458333358168602,
       "step": 241
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 513.0416870117188,
+      "completion_length": 1539.0625610351562,
       "epoch": 0.242,
-      "grad_norm": 0.08471190370306036,
-      "kl": 0.01715087890625,
+      "grad_norm": 4.990238694489094,
+      "kl": 0.94140625,
       "learning_rate": 9.458418577899774e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.2071,
+      "reward": 0.06250000186264515,
+      "reward_std": 0.21650634706020355,
+      "rewards/accuracy_reward": 0.06250000186264515,
       "step": 242
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 575.3333740234375,
+      "completion_length": 1431.1458740234375,
       "epoch": 0.243,
-      "grad_norm": 0.07407780109480006,
-      "kl": 0.014739990234375,
+      "grad_norm": 8.992586270478695,
+      "kl": 0.537109375,
       "learning_rate": 9.450923389425911e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1712,
+      "reward": 0.3333333432674408,
+      "reward_std": 0.2278706133365631,
+      "rewards/accuracy_reward": 0.3333333432674408,
       "step": 243
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 539.125,
+      "completion_length": 1243.6458740234375,
       "epoch": 0.244,
-      "grad_norm": 1.0872363516816363,
-      "kl": 0.01788330078125,
+      "grad_norm": 14.396198414865307,
+      "kl": 0.431640625,
       "learning_rate": 9.443380060197385e-07,
-      "loss": -0.006,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.178,
+      "reward": 0.2708333395421505,
+      "reward_std": 0.16948114335536957,
+      "rewards/accuracy_reward": 0.2708333395421505,
       "step": 244
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 581.9375305175781,
+      "completion_length": 1303.5416870117188,
       "epoch": 0.245,
-      "grad_norm": 0.06841999220752508,
-      "kl": 0.01605224609375,
+      "grad_norm": 10.446802192046018,
+      "kl": 0.818359375,
       "learning_rate": 9.43578868212728e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.2582,
+      "reward": 0.3333333432674408,
+      "reward_std": 0.3157937750220299,
+      "rewards/accuracy_reward": 0.3333333432674408,
       "step": 245
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 546.5208435058594,
+      "completion_length": 1302.1875610351562,
       "epoch": 0.246,
-      "grad_norm": 0.08021586563908664,
-      "kl": 0.01727294921875,
+      "grad_norm": 12.94923238491233,
+      "kl": 0.798828125,
       "learning_rate": 9.428149347714143e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1883,
+      "reward": 0.2916666679084301,
+      "reward_std": 0.29193708300590515,
+      "rewards/accuracy_reward": 0.2916666679084301,
       "step": 246
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 513.2916870117188,
+      "completion_length": 1260.7500610351562,
       "epoch": 0.247,
-      "grad_norm": 1.5957606865057967,
-      "kl": 0.0167236328125,
+      "grad_norm": 10.192159342975343,
+      "kl": 0.48046875,
       "learning_rate": 9.420462150040852e-07,
-      "loss": 0.0147,
-      "reward": 0.9895833730697632,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833730697632,
+      "loss": 0.0309,
+      "reward": 0.1666666716337204,
+      "reward_std": 0.3798602670431137,
+      "rewards/accuracy_reward": 0.1666666716337204,
       "step": 247
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 523.125,
+      "completion_length": 1076.5625610351562,
       "epoch": 0.248,
-      "grad_norm": 0.07161150701962184,
-      "kl": 0.015472412109375,
+      "grad_norm": 23.240852856122473,
+      "kl": 0.552734375,
       "learning_rate": 9.412727182773486e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": -0.0125,
+      "reward": 0.25,
+      "reward_std": 0.4408077299594879,
+      "rewards/accuracy_reward": 0.25,
       "step": 248
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 485.62501525878906,
+      "completion_length": 1069.5208740234375,
       "epoch": 0.249,
-      "grad_norm": 0.07697007954058731,
-      "kl": 0.0164794921875,
+      "grad_norm": 18.185048380387265,
+      "kl": 0.4560546875,
       "learning_rate": 9.404944540160177e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": -0.0291,
+      "reward": 0.3541666865348816,
+      "reward_std": 0.3547166883945465,
+      "rewards/accuracy_reward": 0.3541666865348816,
       "step": 249
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 501.62501525878906,
+      "completion_length": 952.1041870117188,
       "epoch": 0.25,
-      "grad_norm": 0.08389406416336188,
-      "kl": 0.01727294921875,
+      "grad_norm": 58.9276152169841,
+      "kl": 0.66796875,
       "learning_rate": 9.397114317029974e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": -0.2879,
+      "reward": 0.1666666716337204,
+      "reward_std": 0.3798602670431137,
+      "rewards/accuracy_reward": 0.1666666716337204,
       "step": 250
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 487.9375305175781,
+      "completion_length": 961.3750305175781,
       "epoch": 0.251,
-      "grad_norm": 0.0725621860485941,
-      "kl": 0.015411376953125,
+      "grad_norm": 11.922804757705896,
+      "kl": 0.5224609375,
       "learning_rate": 9.38923660879167e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": -0.0463,
+      "reward": 0.5000000298023224,
+      "reward_std": 0.4056393951177597,
+      "rewards/accuracy_reward": 0.5000000298023224,
       "step": 251
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 518.9583435058594,
+      "completion_length": 1326.1041870117188,
       "epoch": 0.252,
-      "grad_norm": 0.08955151459635992,
-      "kl": 0.015899658203125,
+      "grad_norm": 15.797305413372209,
+      "kl": 0.59765625,
       "learning_rate": 9.381311511432658e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.2316,
+      "reward": 0.3541666865348816,
+      "reward_std": 0.4112800657749176,
+      "rewards/accuracy_reward": 0.3541666865348816,
       "step": 252
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 483.35418701171875,
+      "completion_length": 1253.0833740234375,
       "epoch": 0.253,
-      "grad_norm": 0.06650664724173376,
-      "kl": 0.0152587890625,
+      "grad_norm": 10.145906961056761,
+      "kl": 0.466796875,
       "learning_rate": 9.373339121517746e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1268,
+      "reward": 0.10416666977107525,
+      "reward_std": 0.26679353415966034,
+      "rewards/accuracy_reward": 0.10416666977107525,
       "step": 253
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 484.68751525878906,
+      "completion_length": 1525.1041870117188,
       "epoch": 0.254,
-      "grad_norm": 0.08269813073285828,
-      "kl": 0.018310546875,
+      "grad_norm": 11.506301594058074,
+      "kl": 0.517578125,
       "learning_rate": 9.36531953618799e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0914,
+      "reward": 0.2083333395421505,
+      "reward_std": 0.3251829743385315,
+      "rewards/accuracy_reward": 0.2083333395421505,
       "step": 254
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 543.6250305175781,
+      "completion_length": 1161.6250305175781,
       "epoch": 0.255,
-      "grad_norm": 0.057157762620809585,
-      "kl": 0.01470947265625,
+      "grad_norm": 6.368156304855625,
+      "kl": 0.4580078125,
       "learning_rate": 9.357252853159505e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0015,
+      "reward": 0.2708333395421505,
+      "reward_std": 0.3961612284183502,
+      "rewards/accuracy_reward": 0.2708333395421505,
       "step": 255
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 464.5,
+      "completion_length": 1062.3958435058594,
       "epoch": 0.256,
-      "grad_norm": 0.5122966220354302,
-      "kl": 0.02410888671875,
+      "grad_norm": 17.430057095569815,
+      "kl": 0.3642578125,
       "learning_rate": 9.34913917072228e-07,
-      "loss": 0.0009,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0561,
+      "reward": 0.7500000298023224,
+      "reward_std": 0.32399244606494904,
+      "rewards/accuracy_reward": 0.7500000298023224,
       "step": 256
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 485.9375,
+      "completion_length": 1131.2916870117188,
       "epoch": 0.257,
-      "grad_norm": 0.06726881896372686,
-      "kl": 0.01654052734375,
+      "grad_norm": 14.498650490673928,
+      "kl": 0.595703125,
       "learning_rate": 9.340978587738972e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1725,
+      "reward": 0.458333358168602,
+      "reward_std": 0.4665868580341339,
+      "rewards/accuracy_reward": 0.458333358168602,
       "step": 257
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 501.54168701171875,
+      "completion_length": 1203.0833740234375,
       "epoch": 0.258,
-      "grad_norm": 0.0748098899740606,
-      "kl": 0.01788330078125,
+      "grad_norm": 11.196092502910355,
+      "kl": 0.515625,
       "learning_rate": 9.332771203643714e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": -0.0171,
+      "reward": 0.3541666716337204,
+      "reward_std": 0.4370592087507248,
+      "rewards/accuracy_reward": 0.3541666716337204,
       "step": 258
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 488.72918701171875,
+      "completion_length": 1191.7083740234375,
       "epoch": 0.259,
-      "grad_norm": 1.2840455355702072,
-      "kl": 0.01544189453125,
+      "grad_norm": 25.38388672679794,
+      "kl": 0.6875,
       "learning_rate": 9.324517118440888e-07,
-      "loss": 0.0321,
-      "reward": 0.984375,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0486,
+      "reward": 0.7083333432674408,
+      "reward_std": 0.4426998794078827,
+      "rewards/accuracy_reward": 0.7083333432674408,
       "step": 259
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 512.3125305175781,
+      "completion_length": 1278.9791870117188,
       "epoch": 0.26,
-      "grad_norm": 0.05847318018525027,
-      "kl": 0.015411376953125,
+      "grad_norm": 11.012377896726164,
+      "kl": 0.8154296875,
       "learning_rate": 9.316216432703916e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.083,
+      "reward": 0.6041666865348816,
+      "reward_std": 0.4527246206998825,
+      "rewards/accuracy_reward": 0.6041666865348816,
       "step": 260
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 469.1458435058594,
+      "completion_length": 1252.3958740234375,
       "epoch": 0.261,
-      "grad_norm": 1.2796299237266675,
-      "kl": 0.0162353515625,
+      "grad_norm": 8.904251183431551,
+      "kl": 1.095703125,
       "learning_rate": 9.307869247574038e-07,
-      "loss": 0.0266,
-      "reward": 0.984375,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.1808,
+      "reward": 0.6041666865348816,
+      "reward_std": 0.46019136905670166,
+      "rewards/accuracy_reward": 0.6041666865348816,
       "step": 261
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 453.1041717529297,
+      "completion_length": 1300.4792175292969,
       "epoch": 0.262,
-      "grad_norm": 0.0686759440576299,
-      "kl": 0.01666259765625,
+      "grad_norm": 13.746181751786013,
+      "kl": 1.0625,
       "learning_rate": 9.299475664759068e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0037,
+      "reward": 0.5000000298023224,
+      "reward_std": 0.3314591944217682,
+      "rewards/accuracy_reward": 0.5000000298023224,
       "step": 262
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 424.625,
+      "completion_length": 1316.7708740234375,
       "epoch": 0.263,
-      "grad_norm": 1.5060360382856361,
-      "kl": 0.01507568359375,
+      "grad_norm": 13.682572042478128,
+      "kl": 0.810546875,
       "learning_rate": 9.291035786532163e-07,
-      "loss": 0.032,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.1831,
+      "reward": 0.4791666865348816,
+      "reward_std": 0.4370592087507248,
+      "rewards/accuracy_reward": 0.4791666865348816,
       "step": 263
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 499.81251525878906,
+      "completion_length": 1507.979248046875,
       "epoch": 0.264,
-      "grad_norm": 2.449862371398939,
-      "kl": 0.019775390625,
+      "grad_norm": 12.993147908194391,
+      "kl": 1.6796875,
       "learning_rate": 9.282549715730579e-07,
-      "loss": 0.0313,
-      "reward": 0.9687500298023224,
-      "reward_std": 0.10825316980481148,
-      "rewards/tag_count_reward": 0.9687500298023224,
+      "loss": 0.0653,
+      "reward": 0.2500000074505806,
+      "reward_std": 0.3861365020275116,
+      "rewards/accuracy_reward": 0.2500000074505806,
       "step": 264
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 577.3958740234375,
+      "completion_length": 1525.1041870117188,
       "epoch": 0.265,
-      "grad_norm": 0.25453031839063006,
-      "kl": 0.019256591796875,
+      "grad_norm": 15.44157382314789,
+      "kl": 1.26953125,
       "learning_rate": 9.274017555754407e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.198,
+      "reward": 0.458333358168602,
+      "reward_std": 0.4364236444234848,
+      "rewards/accuracy_reward": 0.458333358168602,
       "step": 265
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 455.9166717529297,
+      "completion_length": 1545.6875610351562,
       "epoch": 0.266,
-      "grad_norm": 0.06570987453035185,
-      "kl": 0.015380859375,
+      "grad_norm": 10.825253204085202,
+      "kl": 1.09375,
       "learning_rate": 9.265439410565328e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0962,
+      "reward": 0.2708333432674408,
+      "reward_std": 0.46229176223278046,
+      "rewards/accuracy_reward": 0.2708333432674408,
       "step": 266
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 463.60418701171875,
+      "completion_length": 1495.0833740234375,
       "epoch": 0.267,
-      "grad_norm": 0.07329984438320578,
-      "kl": 0.015380859375,
+      "grad_norm": 17.0572222925967,
+      "kl": 0.865234375,
       "learning_rate": 9.256815384685328e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1256,
+      "reward": 0.458333358168602,
+      "reward_std": 0.41310615837574005,
+      "rewards/accuracy_reward": 0.458333358168602,
       "step": 267
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 467.87501525878906,
+      "completion_length": 1699.7708740234375,
       "epoch": 0.268,
-      "grad_norm": 0.15862357870241905,
-      "kl": 0.014862060546875,
+      "grad_norm": 8.696639388903066,
+      "kl": 1.71875,
       "learning_rate": 9.248145583195447e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1305,
+      "reward": 0.2500000149011612,
+      "reward_std": 0.41310614347457886,
+      "rewards/accuracy_reward": 0.2500000149011612,
       "step": 268
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 426.1041717529297,
+      "completion_length": 1798.979248046875,
       "epoch": 0.269,
-      "grad_norm": 0.06528566167070449,
-      "kl": 0.015899658203125,
+      "grad_norm": 12.792002250815697,
+      "kl": 2.2734375,
       "learning_rate": 9.239430111734476e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.142,
+      "reward": 0.2916666865348816,
+      "reward_std": 0.4723165035247803,
+      "rewards/accuracy_reward": 0.2916666865348816,
       "step": 269
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 426.5833435058594,
+      "completion_length": 1986.7708740234375,
       "epoch": 0.27,
-      "grad_norm": 0.07168963347611128,
-      "kl": 0.01611328125,
+      "grad_norm": 8.75939312098645,
+      "kl": 1.80078125,
       "learning_rate": 9.230669076497687e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0943,
+      "reward": 0.1875000111758709,
+      "reward_std": 0.3334706127643585,
+      "rewards/accuracy_reward": 0.1875000111758709,
       "step": 270
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 523.8750305175781,
+      "completion_length": 1901.4166870117188,
       "epoch": 0.271,
-      "grad_norm": 0.06544622695773468,
-      "kl": 0.01708984375,
+      "grad_norm": 8.88278093314994,
+      "kl": 1.09375,
       "learning_rate": 9.221862584235526e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0738,
+      "reward": 0.3125000149011612,
+      "reward_std": 0.4036279767751694,
+      "rewards/accuracy_reward": 0.3125000149011612,
       "step": 271
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 456.22918701171875,
+      "completion_length": 1996.0208740234375,
       "epoch": 0.272,
-      "grad_norm": 0.062094560268822145,
-      "kl": 0.01666259765625,
+      "grad_norm": 4.723928301568385,
+      "kl": 1.046875,
       "learning_rate": 9.213010742252327e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0522,
+      "reward": 0.08333333395421505,
+      "reward_std": 0.18523553758859634,
+      "rewards/accuracy_reward": 0.08333333395421505,
       "step": 272
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 474.8125,
+      "completion_length": 2011.6041870117188,
       "epoch": 0.273,
-      "grad_norm": 0.05862877618071435,
-      "kl": 0.01605224609375,
+      "grad_norm": 3.6048800109037717,
+      "kl": 0.705078125,
       "learning_rate": 9.204113658404989e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.053,
+      "reward": 0.08333333395421505,
+      "reward_std": 0.24164992570877075,
+      "rewards/accuracy_reward": 0.08333333395421505,
       "step": 273
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 443.9166717529297,
+      "completion_length": 2006.4791870117188,
       "epoch": 0.274,
-      "grad_norm": 0.06164062190782835,
-      "kl": 0.015228271484375,
+      "grad_norm": 4.237553704215963,
+      "kl": 0.5380859375,
       "learning_rate": 9.195171441101668e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0449,
+      "reward": 0.06250000186264515,
+      "reward_std": 0.16948114335536957,
+      "rewards/accuracy_reward": 0.06250000186264515,
       "step": 274
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 457.1041717529297,
+      "completion_length": 2027.604248046875,
       "epoch": 0.275,
-      "grad_norm": 0.05589128390148594,
-      "kl": 0.014068603515625,
+      "grad_norm": 4.313599501553088,
+      "kl": 0.5419921875,
       "learning_rate": 9.186184199300463e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0314,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.14433756470680237,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 275
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 497.10418701171875,
+      "completion_length": 2025.0000610351562,
       "epoch": 0.276,
-      "grad_norm": 0.05590537885725384,
-      "kl": 0.014373779296875,
+      "grad_norm": 5.138700086850159,
+      "kl": 0.685546875,
       "learning_rate": 9.177152042508077e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0519,
+      "reward": 0.1041666679084301,
+      "reward_std": 0.25740431994199753,
+      "rewards/accuracy_reward": 0.1041666679084301,
       "step": 276
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 486.8125305175781,
+      "completion_length": 1986.2708740234375,
       "epoch": 0.277,
-      "grad_norm": 0.055203648657737074,
-      "kl": 0.01580810546875,
+      "grad_norm": 4.9229467225055785,
+      "kl": 0.5810546875,
       "learning_rate": 9.168075080778494e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0708,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 277
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 451.3958435058594,
+      "completion_length": 2025.9375,
       "epoch": 0.278,
-      "grad_norm": 0.054423924961094675,
-      "kl": 0.015106201171875,
+      "grad_norm": 10.371232301602973,
+      "kl": 0.4638671875,
       "learning_rate": 9.158953424711624e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0186,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.14433756470680237,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 278
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 498.6250305175781,
+      "completion_length": 2021.9791870117188,
       "epoch": 0.279,
-      "grad_norm": 0.04977983112824281,
-      "kl": 0.015594482421875,
+      "grad_norm": 4.300248069423385,
+      "kl": 0.5673828125,
       "learning_rate": 9.149787185451969e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0435,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 279
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 515.875,
+      "completion_length": 2048.0,
       "epoch": 0.28,
-      "grad_norm": 1.5180700680436665,
-      "kl": 0.014678955078125,
+      "grad_norm": 2.1407753380159096,
+      "kl": 0.4599609375,
       "learning_rate": 9.140576474687263e-07,
-      "loss": 0.0476,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0184,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 280
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 501.7708435058594,
+      "completion_length": 2042.1458740234375,
       "epoch": 0.281,
-      "grad_norm": 0.05324506592064358,
-      "kl": 0.02032470703125,
+      "grad_norm": 5.333859123342021,
+      "kl": 0.4033203125,
       "learning_rate": 9.131321404647109e-07,
-      "loss": 0.0008,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0218,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.09731236100196838,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 281
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 433.6666717529297,
+      "completion_length": 2028.5833740234375,
       "epoch": 0.282,
-      "grad_norm": 0.0563429105599599,
-      "kl": 0.01556396484375,
+      "grad_norm": 4.831609443125731,
+      "kl": 0.3349609375,
       "learning_rate": 9.122022088101613e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0229,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 282
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 504.18751525878906,
+      "completion_length": 2048.0,
       "epoch": 0.283,
-      "grad_norm": 0.04710493186631686,
-      "kl": 0.01336669921875,
+      "grad_norm": 3.7141746807665377,
+      "kl": 0.3369140625,
       "learning_rate": 9.112678638360015e-07,
-      "loss": 0.0005,
-      "reward": 1.0,
+      "loss": 0.0135,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 283
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 436.2916717529297,
+      "completion_length": 2048.0,
       "epoch": 0.284,
-      "grad_norm": 0.04673067805273048,
-      "kl": 0.014251708984375,
+      "grad_norm": 2.5426872465697263,
+      "kl": 0.259765625,
       "learning_rate": 9.103291169269299e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
+      "loss": 0.0104,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 284
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 415.6458435058594,
+      "completion_length": 2029.7916870117188,
       "epoch": 0.285,
-      "grad_norm": 0.05669767173984359,
-      "kl": 0.015106201171875,
+      "grad_norm": 5.692035018910782,
+      "kl": 0.24072265625,
       "learning_rate": 9.093859795212817e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0075,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.14433756470680237,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 285
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 444.43751525878906,
+      "completion_length": 2035.7291870117188,
       "epoch": 0.286,
-      "grad_norm": 0.05594767081328561,
-      "kl": 0.01556396484375,
+      "grad_norm": 1.3384672726753144,
+      "kl": 0.22021484375,
       "learning_rate": 9.084384631108882e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
+      "loss": 0.0088,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 286
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 439.75001525878906,
+      "completion_length": 2048.0,
       "epoch": 0.287,
-      "grad_norm": 0.07322331204423045,
-      "kl": 0.013763427734375,
+      "grad_norm": 0.4388542471358353,
+      "kl": 0.21826171875,
       "learning_rate": 9.074865792409381e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
+      "loss": 0.0087,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 287
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 469.0625,
+      "completion_length": 2048.0,
       "epoch": 0.288,
-      "grad_norm": 1.212173860336188,
-      "kl": 0.0167236328125,
+      "grad_norm": 0.9242559510295835,
+      "kl": 0.26953125,
       "learning_rate": 9.065303395098358e-07,
-      "loss": -0.0004,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0108,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 288
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 376.6875,
+      "completion_length": 2048.0,
       "epoch": 0.289,
-      "grad_norm": 0.10599496620311394,
-      "kl": 0.015869140625,
+      "grad_norm": 1.3252684718503742,
+      "kl": 0.3173828125,
       "learning_rate": 9.055697555690607e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
+      "loss": 0.0127,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 289
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 511.2083435058594,
+      "completion_length": 2048.0,
       "epoch": 0.29,
-      "grad_norm": 0.04930889731429071,
-      "kl": 0.014617919921875,
+      "grad_norm": 1.1996743043796287,
+      "kl": 0.2373046875,
       "learning_rate": 9.046048391230247e-07,
-      "loss": 0.0005,
-      "reward": 1.0,
+      "loss": 0.0095,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 290
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 457.0416717529297,
+      "completion_length": 2048.0,
       "epoch": 0.291,
-      "grad_norm": 0.05112757445066342,
-      "kl": 0.01251220703125,
+      "grad_norm": 1.0344002959602598,
+      "kl": 0.19580078125,
       "learning_rate": 9.036356019289309e-07,
-      "loss": 0.0005,
-      "reward": 1.0,
+      "loss": 0.0078,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 291
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 461.3541717529297,
+      "completion_length": 2026.9583740234375,
       "epoch": 0.292,
-      "grad_norm": 0.06788187986805701,
-      "kl": 0.0150146484375,
+      "grad_norm": 0.4611132338447776,
+      "kl": 0.15869140625,
       "learning_rate": 9.026620557966279e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
+      "loss": 0.0064,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 292
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 443.93751525878906,
+      "completion_length": 2048.0,
       "epoch": 0.293,
-      "grad_norm": 0.06684481261288397,
-      "kl": 0.016510009765625,
+      "grad_norm": 1.9536673547974641,
+      "kl": 0.1484375,
       "learning_rate": 9.016842125884684e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
+      "loss": 0.0059,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 293
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 461.7291717529297,
+      "completion_length": 2048.0,
       "epoch": 0.294,
-      "grad_norm": 0.05151572062341038,
-      "kl": 0.01513671875,
+      "grad_norm": 1.6952441884684657,
+      "kl": 0.1357421875,
       "learning_rate": 9.007020842191634e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
+      "loss": 0.0054,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 294
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 503.5208435058594,
+      "completion_length": 2048.0,
       "epoch": 0.295,
-      "grad_norm": 0.0590936047141315,
-      "kl": 0.013519287109375,
+      "grad_norm": 0.8895945755235983,
+      "kl": 0.127197265625,
       "learning_rate": 8.997156826556369e-07,
-      "loss": 0.0005,
-      "reward": 1.0,
+      "loss": 0.0051,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 295
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 545.0625152587891,
+      "completion_length": 2048.0,
       "epoch": 0.296,
-      "grad_norm": 0.05288289391515388,
-      "kl": 0.01641845703125,
+      "grad_norm": 0.634162725119018,
+      "kl": 0.1318359375,
       "learning_rate": 8.987250199168808e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
+      "loss": 0.0053,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 296
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 517.7291717529297,
+      "completion_length": 2048.0,
       "epoch": 0.297,
-      "grad_norm": 0.060860807391451464,
-      "kl": 0.016143798828125,
+      "grad_norm": 0.8559590451608104,
+      "kl": 0.132568359375,
       "learning_rate": 8.977301080738079e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
+      "loss": 0.0053,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 297
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 473.3958435058594,
+      "completion_length": 2027.6875,
       "epoch": 0.298,
-      "grad_norm": 0.05173550443403316,
-      "kl": 0.017578125,
+      "grad_norm": 0.849603096625076,
+      "kl": 0.12353515625,
       "learning_rate": 8.967309592491052e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
+      "loss": 0.0049,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 298
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 501.0208435058594,
+      "completion_length": 2048.0,
       "epoch": 0.299,
-      "grad_norm": 1.7229744630333605,
-      "kl": 0.014373779296875,
+      "grad_norm": 0.31351623243473703,
+      "kl": 0.0986328125,
       "learning_rate": 8.957275856170855e-07,
-      "loss": -0.0001,
-      "reward": 0.9895833730697632,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833730697632,
+      "loss": 0.0039,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 299
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 517.5208435058594,
+      "completion_length": 2048.0,
       "epoch": 0.3,
-      "grad_norm": 0.054848896917061254,
-      "kl": 0.0145263671875,
+      "grad_norm": 0.19523876678801383,
+      "kl": 0.095947265625,
       "learning_rate": 8.9471999940354e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
+      "loss": 0.0038,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 300
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 473.4791717529297,
+      "completion_length": 2011.0,
       "epoch": 0.301,
-      "grad_norm": 1.1842846408033978,
-      "kl": 0.01910400390625,
+      "grad_norm": 0.35217706480459204,
+      "kl": 0.09765625,
       "learning_rate": 8.937082128855891e-07,
-      "loss": 0.0054,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0039,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 301
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 498.5625,
+      "completion_length": 2046.125,
       "epoch": 0.302,
-      "grad_norm": 0.0416267862511951,
-      "kl": 0.01568603515625,
+      "grad_norm": 0.5657716182919389,
+      "kl": 0.096923828125,
       "learning_rate": 8.926922383915315e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
+      "loss": 0.0039,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 302
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 467.87501525878906,
+      "completion_length": 2048.0,
       "epoch": 0.303,
-      "grad_norm": 0.04731658896726055,
-      "kl": 0.01678466796875,
+      "grad_norm": 0.5134333227675091,
+      "kl": 0.09375,
       "learning_rate": 8.916720883006963e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
+      "loss": 0.0037,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 303
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 526.2291870117188,
+      "completion_length": 2048.0,
       "epoch": 0.304,
-      "grad_norm": 1.2303805096767035,
-      "kl": 0.016326904296875,
+      "grad_norm": 0.31198120519038114,
+      "kl": 0.0859375,
       "learning_rate": 8.906477750432903e-07,
-      "loss": 0.0044,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0034,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 304
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 579.3333435058594,
+      "completion_length": 2048.0,
       "epoch": 0.305,
-      "grad_norm": 0.05744770348848511,
-      "kl": 0.015533447265625,
+      "grad_norm": 0.22122977732843133,
+      "kl": 0.080322265625,
       "learning_rate": 8.896193111002475e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
+      "loss": 0.0032,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 305
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 427.3125,
+      "completion_length": 2041.8541870117188,
       "epoch": 0.306,
-      "grad_norm": 0.05850889841453987,
-      "kl": 0.01361083984375,
+      "grad_norm": 0.12183863412385945,
+      "kl": 0.076171875,
       "learning_rate": 8.88586709003076e-07,
-      "loss": 0.0005,
-      "reward": 1.0,
+      "loss": 0.003,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 306
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 470.9166717529297,
+      "completion_length": 2013.25,
       "epoch": 0.307,
-      "grad_norm": 1.1348449824926166,
-      "kl": 0.016357421875,
+      "grad_norm": 7.081667267286064,
+      "kl": 0.077880859375,
       "learning_rate": 8.875499813337067e-07,
-      "loss": 0.0164,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0602,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 307
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 455.3541717529297,
+      "completion_length": 2011.625,
       "epoch": 0.308,
-      "grad_norm": 0.06661742406273104,
-      "kl": 0.01873779296875,
+      "grad_norm": 0.2283109706831473,
+      "kl": 0.073974609375,
       "learning_rate": 8.865091407243394e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
+      "loss": 0.0029,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 308
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 458.85418701171875,
+      "completion_length": 2016.0625,
       "epoch": 0.309,
-      "grad_norm": 0.05716761421750185,
-      "kl": 0.01800537109375,
+      "grad_norm": 6.68469769379313,
+      "kl": 0.077880859375,
       "learning_rate": 8.85464199857288e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0552,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 309
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 495.95835876464844,
+      "completion_length": 2042.6666870117188,
       "epoch": 0.31,
-      "grad_norm": 0.06628759135865589,
-      "kl": 0.01934814453125,
+      "grad_norm": 0.5606966865197962,
+      "kl": 0.080810546875,
       "learning_rate": 8.844151714648274e-07,
-      "loss": 0.0008,
-      "reward": 1.0,
+      "loss": 0.0032,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 310
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 493.7083435058594,
+      "completion_length": 2048.0,
       "epoch": 0.311,
-      "grad_norm": 0.07749485923013408,
-      "kl": 0.01983642578125,
+      "grad_norm": 0.800796312280849,
+      "kl": 0.084228515625,
       "learning_rate": 8.833620683290375e-07,
-      "loss": 0.0008,
-      "reward": 1.0,
+      "loss": 0.0034,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 311
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 512.4375305175781,
+      "completion_length": 2048.0,
       "epoch": 0.312,
-      "grad_norm": 0.09232922199419766,
-      "kl": 0.01904296875,
+      "grad_norm": 0.5932387886813967,
+      "kl": 0.0771484375,
       "learning_rate": 8.823049032816478e-07,
-      "loss": 0.0008,
-      "reward": 1.0,
+      "loss": 0.0031,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 312
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 507.0833435058594,
+      "completion_length": 2014.2708740234375,
       "epoch": 0.313,
-      "grad_norm": 0.0710841483167992,
-      "kl": 0.02093505859375,
+      "grad_norm": 5.642587682583222,
+      "kl": 0.074462890625,
       "learning_rate": 8.812436892038805e-07,
-      "loss": 0.0009,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0583,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 313
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 459.4583435058594,
+      "completion_length": 2013.3333740234375,
       "epoch": 0.314,
-      "grad_norm": 0.3275093589318186,
-      "kl": 0.018798828125,
+      "grad_norm": 0.41895740623439276,
+      "kl": 0.072509765625,
       "learning_rate": 8.801784390262943e-07,
-      "loss": 0.0008,
-      "reward": 1.0,
+      "loss": 0.0029,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 314
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 526.5625152587891,
+      "completion_length": 2048.0,
       "epoch": 0.315,
-      "grad_norm": 0.059675318673412456,
-      "kl": 0.015472412109375,
+      "grad_norm": 0.5503249851060379,
+      "kl": 0.075927734375,
       "learning_rate": 8.791091657286267e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
+      "loss": 0.003,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 315
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 463.00001525878906,
+      "completion_length": 2010.6875,
       "epoch": 0.316,
-      "grad_norm": 0.13956820712764417,
-      "kl": 0.02545166015625,
+      "grad_norm": 0.13839662527380464,
+      "kl": 0.0634765625,
       "learning_rate": 8.780358823396352e-07,
-      "loss": 0.001,
-      "reward": 1.0,
+      "loss": 0.0025,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 316
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 514.125,
+      "completion_length": 1960.7916870117188,
       "epoch": 0.317,
-      "grad_norm": 0.06831673248504883,
-      "kl": 0.01904296875,
+      "grad_norm": 10.575475904350021,
+      "kl": 0.06201171875,
       "learning_rate": 8.769586019369391e-07,
-      "loss": 0.0008,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1149,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.14433756470680237,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 317
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 523.4583587646484,
+      "completion_length": 1979.4583740234375,
       "epoch": 0.318,
-      "grad_norm": 0.15825854582183677,
-      "kl": 0.01904296875,
+      "grad_norm": 0.12922184489679245,
+      "kl": 0.0611572265625,
       "learning_rate": 8.758773376468604e-07,
-      "loss": 0.0008,
-      "reward": 1.0,
+      "loss": 0.0024,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 318
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 482.72918701171875,
+      "completion_length": 2048.0,
       "epoch": 0.319,
-      "grad_norm": 0.053797281866110686,
-      "kl": 0.0186767578125,
+      "grad_norm": 0.24866797691358314,
+      "kl": 0.061279296875,
       "learning_rate": 8.747921026442629e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
+      "loss": 0.0025,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 319
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 502.6458435058594,
+      "completion_length": 2046.8333740234375,
       "epoch": 0.32,
-      "grad_norm": 0.04354466180020813,
-      "kl": 0.015777587890625,
+      "grad_norm": 0.07895714717603564,
+      "kl": 0.0584716796875,
       "learning_rate": 8.737029101523929e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
+      "loss": 0.0023,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 320
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 542.9583587646484,
+      "completion_length": 2011.9791870117188,
       "epoch": 0.321,
-      "grad_norm": 0.04820768922800098,
-      "kl": 0.015869140625,
+      "grad_norm": 4.734647110583716,
+      "kl": 0.0565185546875,
       "learning_rate": 8.726097734427172e-07,
-      "loss": 0.0006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0577,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 321
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 502.04168701171875,
+      "completion_length": 1979.1666870117188,
       "epoch": 0.322,
-      "grad_norm": 0.059999763448746204,
-      "kl": 0.01849365234375,
+      "grad_norm": 4.489754741724055,
+      "kl": 0.0631103515625,
       "learning_rate": 8.715127058347614e-07,
-      "loss": 0.0007,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0467,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 322
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 492.37501525878906,
+      "completion_length": 1976.7500610351562,
       "epoch": 0.323,
-      "grad_norm": 0.43781729225200805,
-      "kl": 0.02587890625,
+      "grad_norm": 0.6681220339187135,
+      "kl": 0.0712890625,
       "learning_rate": 8.704117206959484e-07,
-      "loss": 0.0011,
-      "reward": 1.0,
+      "loss": 0.0029,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 323
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 505.4791717529297,
+      "completion_length": 1887.3958740234375,
       "epoch": 0.324,
-      "grad_norm": 0.05736039229627184,
-      "kl": 0.0220947265625,
+      "grad_norm": 9.631400959130877,
+      "kl": 0.079833984375,
       "learning_rate": 8.693068314414344e-07,
-      "loss": 0.0009,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.109,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.14433756470680237,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 324
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 510.1250305175781,
+      "completion_length": 1990.75,
       "epoch": 0.325,
-      "grad_norm": 0.05452839261024288,
-      "kl": 0.020263671875,
+      "grad_norm": 4.360699152638972,
+      "kl": 0.1318359375,
       "learning_rate": 8.681980515339463e-07,
-      "loss": 0.0008,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0635,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 325
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 535.3333740234375,
+      "completion_length": 1916.354248046875,
       "epoch": 0.326,
-      "grad_norm": 0.8837300899462217,
-      "kl": 0.018035888671875,
+      "grad_norm": 6.198221310120352,
+      "kl": 0.154296875,
       "learning_rate": 8.670853944836176e-07,
-      "loss": -0.0116,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0813,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.09731236100196838,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 326
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 586.9375305175781,
+      "completion_length": 1845.0208740234375,
       "epoch": 0.327,
-      "grad_norm": 1.6811219122897223,
-      "kl": 0.02203369140625,
+      "grad_norm": 4.498082048090132,
+      "kl": 0.1884765625,
       "learning_rate": 8.659688738478231e-07,
-      "loss": 0.1342,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.07216878235340118,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0718,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.09731236100196838,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 327
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 489.3333435058594,
+      "completion_length": 1792.1666870117188,
       "epoch": 0.328,
-      "grad_norm": 1.8690265977611447,
-      "kl": 0.025390625,
+      "grad_norm": 9.05737548350279,
+      "kl": 0.275390625,
       "learning_rate": 8.648485032310144e-07,
-      "loss": 0.0393,
-      "reward": 0.9895833730697632,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833730697632,
+      "loss": 0.1334,
+      "reward": 0.1041666679084301,
+      "reward_std": 0.21037911623716354,
+      "rewards/accuracy_reward": 0.1041666679084301,
       "step": 328
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 509.12501525878906,
+      "completion_length": 1869.75,
       "epoch": 0.329,
-      "grad_norm": 2.034164201051351,
-      "kl": 0.0216064453125,
+      "grad_norm": 3.5258311968745786,
+      "kl": 0.443359375,
       "learning_rate": 8.63724296284554e-07,
-      "loss": 0.1843,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.07216878235340118,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0904,
+      "reward": 0.0625,
+      "reward_std": 0.16948114335536957,
+      "rewards/accuracy_reward": 0.0625,
       "step": 329
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 444.7916717529297,
+      "completion_length": 1446.9583740234375,
       "epoch": 0.33,
-      "grad_norm": 0.05567755862652329,
-      "kl": 0.0224609375,
+      "grad_norm": 25.042200088167174,
+      "kl": 0.5078125,
       "learning_rate": 8.625962667065487e-07,
-      "loss": 0.0009,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.316,
+      "reward": 0.2083333358168602,
+      "reward_std": 0.4056393951177597,
+      "rewards/accuracy_reward": 0.2083333358168602,
       "step": 330
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 581.4166870117188,
+      "completion_length": 1852.3958740234375,
       "epoch": 0.331,
-      "grad_norm": 0.04654044343912161,
-      "kl": 0.01922607421875,
+      "grad_norm": 4.613287464945837,
+      "kl": 0.771484375,
       "learning_rate": 8.614644282416831e-07,
-      "loss": 0.0008,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1474,
+      "reward": 0.0625,
+      "reward_std": 0.16948114335536957,
+      "rewards/accuracy_reward": 0.0625,
       "step": 331
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 494.9375,
+      "completion_length": 1602.8333740234375,
       "epoch": 0.332,
-      "grad_norm": 0.06383138430755644,
-      "kl": 0.0205078125,
+      "grad_norm": 16.933924420353947,
+      "kl": 0.9375,
       "learning_rate": 8.603287946810513e-07,
-      "loss": 0.0008,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.347,
+      "reward": 0.1666666716337204,
+      "reward_std": 0.3798602670431137,
+      "rewards/accuracy_reward": 0.1666666716337204,
       "step": 332
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 472.4166717529297,
+      "completion_length": 1310.3958740234375,
       "epoch": 0.333,
-      "grad_norm": 0.09659644276035662,
-      "kl": 0.02178955078125,
+      "grad_norm": 15.781379569835412,
+      "kl": 1.001953125,
       "learning_rate": 8.591893798619903e-07,
-      "loss": 0.0009,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.3603,
+      "reward": 0.25,
+      "reward_std": 0.364890418946743,
+      "rewards/accuracy_reward": 0.25,
       "step": 333
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 482.52085876464844,
+      "completion_length": 1423.8750610351562,
       "epoch": 0.334,
-      "grad_norm": 0.0633386796003193,
-      "kl": 0.02069091796875,
+      "grad_norm": 16.155552799852583,
+      "kl": 1.375,
       "learning_rate": 8.580461976679099e-07,
-      "loss": 0.0008,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.4276,
+      "reward": 0.2916666716337204,
+      "reward_std": 0.4213048219680786,
+      "rewards/accuracy_reward": 0.2916666716337204,
       "step": 334
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 507.1041717529297,
+      "completion_length": 1272.1666870117188,
       "epoch": 0.335,
-      "grad_norm": 1.1750271881166172,
-      "kl": 0.024169921875,
+      "grad_norm": 9.797362078873267,
+      "kl": 1.59375,
       "learning_rate": 8.568992620281243e-07,
-      "loss": 0.0184,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.3602,
+      "reward": 0.2708333358168602,
+      "reward_std": 0.42066924273967743,
+      "rewards/accuracy_reward": 0.2708333358168602,
       "step": 335
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 515.2083435058594,
+      "completion_length": 1502.5416870117188,
       "epoch": 0.336,
-      "grad_norm": 0.0705503835518815,
-      "kl": 0.02325439453125,
+      "grad_norm": 4.021906077845837,
+      "kl": 2.7265625,
       "learning_rate": 8.557485869176825e-07,
-      "loss": 0.0009,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.4186,
+      "reward": 0.229166679084301,
+      "reward_std": 0.3592497557401657,
+      "rewards/accuracy_reward": 0.229166679084301,
       "step": 336
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 526.1041870117188,
+      "completion_length": 1381.625,
       "epoch": 0.337,
-      "grad_norm": 0.10934531694748963,
-      "kl": 0.0225830078125,
+      "grad_norm": 5.872128666756936,
+      "kl": 2.9453125,
       "learning_rate": 8.545941863571973e-07,
-      "loss": 0.0009,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.4131,
+      "reward": 0.229166679084301,
+      "reward_std": 0.38796256482601166,
+      "rewards/accuracy_reward": 0.229166679084301,
       "step": 337
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 494.85418701171875,
+      "completion_length": 1376.5833740234375,
       "epoch": 0.338,
-      "grad_norm": 0.06572627733993577,
-      "kl": 0.0233154296875,
+      "grad_norm": 9.25490552852953,
+      "kl": 3.46875,
       "learning_rate": 8.534360744126753e-07,
-      "loss": 0.0009,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.4431,
+      "reward": 0.2291666716337204,
+      "reward_std": 0.4156641513109207,
+      "rewards/accuracy_reward": 0.2291666716337204,
       "step": 338
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 470.47918701171875,
+      "completion_length": 1410.7291870117188,
       "epoch": 0.339,
-      "grad_norm": 0.06859987776775232,
-      "kl": 0.0250244140625,
+      "grad_norm": 11.093609314856238,
+      "kl": 3.5703125,
       "learning_rate": 8.522742651953456e-07,
-      "loss": 0.001,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.4589,
+      "reward": 0.2708333432674408,
+      "reward_std": 0.4565621018409729,
+      "rewards/accuracy_reward": 0.2708333432674408,
       "step": 339
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 544.3750305175781,
+      "completion_length": 1384.1458740234375,
       "epoch": 0.34,
-      "grad_norm": 0.06359343320977029,
-      "kl": 0.0220947265625,
+      "grad_norm": 13.648606837019173,
+      "kl": 3.25,
       "learning_rate": 8.511087728614862e-07,
-      "loss": 0.0009,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.3663,
+      "reward": 0.4166666716337204,
+      "reward_std": 0.48533491790294647,
+      "rewards/accuracy_reward": 0.4166666716337204,
       "step": 340
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 491.3541717529297,
+      "completion_length": 1548.0208740234375,
       "epoch": 0.341,
-      "grad_norm": 0.08259686589085237,
-      "kl": 0.02557373046875,
+      "grad_norm": 17.782336463551186,
+      "kl": 3.7734375,
       "learning_rate": 8.499396116122535e-07,
-      "loss": 0.001,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.2519,
+      "reward": 0.2916666716337204,
+      "reward_std": 0.35477669537067413,
+      "rewards/accuracy_reward": 0.2916666716337204,
       "step": 341
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 484.1458435058594,
+      "completion_length": 1695.2916870117188,
       "epoch": 0.342,
-      "grad_norm": 0.09912480037756079,
-      "kl": 0.02545166015625,
+      "grad_norm": 10.719218585105253,
+      "kl": 3.3984375,
       "learning_rate": 8.487667956935087e-07,
-      "loss": 0.001,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.2093,
+      "reward": 0.5208333432674408,
+      "reward_std": 0.5167547464370728,
+      "rewards/accuracy_reward": 0.5208333432674408,
       "step": 342
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 469.2708435058594,
+      "completion_length": 1882.8958740234375,
       "epoch": 0.343,
-      "grad_norm": 0.07102065021267147,
-      "kl": 0.025390625,
+      "grad_norm": 4.504280206152368,
+      "kl": 3.078125,
       "learning_rate": 8.475903393956433e-07,
-      "loss": 0.001,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.2084,
+      "reward": 0.1041666679084301,
+      "reward_std": 0.26679350435733795,
+      "rewards/accuracy_reward": 0.1041666679084301,
       "step": 343
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 533.7083740234375,
+      "completion_length": 1958.1875610351562,
       "epoch": 0.344,
-      "grad_norm": 1.1653009469800304,
-      "kl": 0.02618408203125,
+      "grad_norm": 4.592443313603408,
+      "kl": 3.0546875,
       "learning_rate": 8.464102570534061e-07,
-      "loss": 0.0184,
-      "reward": 0.984375,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.147,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.14433756470680237,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 344
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 518.3750152587891,
+      "completion_length": 2013.4375610351562,
       "epoch": 0.345,
-      "grad_norm": 0.0967071028899652,
-      "kl": 0.0279541015625,
+      "grad_norm": 5.062322567617267,
+      "kl": 3.1484375,
       "learning_rate": 8.452265630457282e-07,
-      "loss": 0.0011,
-      "reward": 1.0,
+      "loss": 0.1262,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 345
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 601.8958435058594,
+      "completion_length": 2006.7083740234375,
       "epoch": 0.346,
-      "grad_norm": 1.1680093291977525,
-      "kl": 0.02410888671875,
+      "grad_norm": 1.3444678791595068,
+      "kl": 2.7890625,
       "learning_rate": 8.440392717955475e-07,
-      "loss": 0.0125,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.1139,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 346
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 480.7083435058594,
+      "completion_length": 2044.75,
       "epoch": 0.347,
-      "grad_norm": 0.0684180862342716,
-      "kl": 0.02630615234375,
+      "grad_norm": 0.6930600957305891,
+      "kl": 2.8203125,
       "learning_rate": 8.428483977696328e-07,
-      "loss": 0.001,
-      "reward": 1.0,
+      "loss": 0.1129,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 347
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 517.8958435058594,
+      "completion_length": 2005.604248046875,
       "epoch": 0.348,
-      "grad_norm": 1.1491175892311671,
-      "kl": 0.0286865234375,
+      "grad_norm": 1.3773757263575364,
+      "kl": 2.90625,
       "learning_rate": 8.416539554784089e-07,
-      "loss": 0.0028,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.135,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.09731236100196838,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 348
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 473.3958435058594,
+      "completion_length": 2042.3958740234375,
       "epoch": 0.349,
-      "grad_norm": 0.9962984772350725,
-      "kl": 0.02545166015625,
+      "grad_norm": 1.1717368391814287,
+      "kl": 2.6171875,
       "learning_rate": 8.404559594757777e-07,
-      "loss": -0.0051,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.1129,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 349
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 511.00001525878906,
+      "completion_length": 1873.6666870117188,
       "epoch": 0.35,
-      "grad_norm": 0.07845658168559723,
-      "kl": 0.0296630859375,
+      "grad_norm": 3.489885564070273,
+      "kl": 2.25,
       "learning_rate": 8.392544243589427e-07,
-      "loss": 0.0012,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1333,
+      "reward": 0.3750000149011612,
+      "reward_std": 0.46966953575611115,
+      "rewards/accuracy_reward": 0.3750000149011612,
       "step": 350
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 525.7708435058594,
+      "completion_length": 1696.6666870117188,
       "epoch": 0.351,
-      "grad_norm": 0.9282118009818712,
-      "kl": 0.031005859375,
+      "grad_norm": 4.413344103304236,
+      "kl": 1.88671875,
       "learning_rate": 8.3804936476823e-07,
-      "loss": 0.0192,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.1411,
+      "reward": 0.6041666865348816,
+      "reward_std": 0.4936225861310959,
+      "rewards/accuracy_reward": 0.6041666865348816,
       "step": 351
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 563.8750152587891,
+      "completion_length": 1561.7708740234375,
       "epoch": 0.352,
-      "grad_norm": 0.06552065313261518,
-      "kl": 0.02557373046875,
+      "grad_norm": 11.380484541507656,
+      "kl": 1.82421875,
       "learning_rate": 8.368407953869103e-07,
-      "loss": 0.001,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1231,
+      "reward": 0.291666679084301,
+      "reward_std": 0.44827449321746826,
+      "rewards/accuracy_reward": 0.291666679084301,
       "step": 352
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 581.0625305175781,
+      "completion_length": 1605.0625,
       "epoch": 0.353,
-      "grad_norm": 0.060361965526579735,
-      "kl": 0.02569580078125,
+      "grad_norm": 14.55730631917554,
+      "kl": 1.4140625,
       "learning_rate": 8.356287309410204e-07,
-      "loss": 0.001,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0808,
+      "reward": 0.375,
+      "reward_std": 0.4872499853372574,
+      "rewards/accuracy_reward": 0.375,
       "step": 353
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 548.6666717529297,
+      "completion_length": 1398.6250610351562,
       "epoch": 0.354,
-      "grad_norm": 0.10004136159227021,
-      "kl": 0.03033447265625,
+      "grad_norm": 13.445064472119112,
+      "kl": 1.3203125,
       "learning_rate": 8.344131861991828e-07,
-      "loss": 0.0012,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1511,
+      "reward": 0.6041666716337204,
+      "reward_std": 0.46019135415554047,
+      "rewards/accuracy_reward": 0.6041666716337204,
       "step": 354
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 570.4583587646484,
+      "completion_length": 1585.8958740234375,
       "epoch": 0.355,
-      "grad_norm": 0.07661012404336875,
-      "kl": 0.0269775390625,
+      "grad_norm": 17.643779145482224,
+      "kl": 1.50390625,
       "learning_rate": 8.331941759724268e-07,
-      "loss": 0.0011,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1043,
+      "reward": 0.3333333432674408,
+      "reward_std": 0.4879819303750992,
+      "rewards/accuracy_reward": 0.3333333432674408,
       "step": 355
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 585.1250305175781,
+      "completion_length": 1620.9166870117188,
       "epoch": 0.356,
-      "grad_norm": 0.06700631082240648,
-      "kl": 0.02691650390625,
+      "grad_norm": 8.667941124357887,
+      "kl": 1.35546875,
       "learning_rate": 8.319717151140072e-07,
-      "loss": 0.0011,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1607,
+      "reward": 0.4375,
+      "reward_std": 0.498006671667099,
+      "rewards/accuracy_reward": 0.4375,
       "step": 356
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 559.2708587646484,
+      "completion_length": 1713.1250610351562,
       "epoch": 0.357,
-      "grad_norm": 1.1262725311155874,
-      "kl": 0.02838134765625,
+      "grad_norm": 14.877019583504781,
+      "kl": 1.359375,
       "learning_rate": 8.307458185192238e-07,
-      "loss": -0.0009,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.2112,
+      "reward": 0.3750000149011612,
+      "reward_std": 0.4815203845500946,
+      "rewards/accuracy_reward": 0.3750000149011612,
       "step": 357
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 580.2500305175781,
+      "completion_length": 1898.5833740234375,
       "epoch": 0.358,
-      "grad_norm": 0.07699460992208937,
-      "kl": 0.028564453125,
+      "grad_norm": 11.018475356178357,
+      "kl": 1.578125,
       "learning_rate": 8.295165011252396e-07,
-      "loss": 0.0012,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1329,
+      "reward": 0.1875,
+      "reward_std": 0.3703821152448654,
+      "rewards/accuracy_reward": 0.1875,
       "step": 358
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 488.4166717529297,
+      "completion_length": 1898.541748046875,
       "epoch": 0.359,
-      "grad_norm": 0.07259581116856176,
-      "kl": 0.0335693359375,
+      "grad_norm": 12.501083842873744,
+      "kl": 1.53515625,
       "learning_rate": 8.282837779108993e-07,
-      "loss": 0.0013,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0907,
+      "reward": 0.1666666679084301,
+      "reward_std": 0.36474141478538513,
+      "rewards/accuracy_reward": 0.1666666679084301,
       "step": 359
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 601.1041870117188,
+      "completion_length": 1975.1666870117188,
       "epoch": 0.36,
-      "grad_norm": 1.645882079495026,
-      "kl": 0.03192138671875,
+      "grad_norm": 2.684893299505885,
+      "kl": 1.25,
       "learning_rate": 8.270476638965461e-07,
-      "loss": 0.1289,
-      "reward": 0.984375,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.1236,
+      "reward": 0.06250000186264515,
+      "reward_std": 0.16948114335536957,
+      "rewards/accuracy_reward": 0.06250000186264515,
       "step": 360
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 604.5208435058594,
+      "completion_length": 2026.7500610351562,
       "epoch": 0.361,
-      "grad_norm": 0.06797167589286383,
-      "kl": 0.02838134765625,
+      "grad_norm": 7.155970369722617,
+      "kl": 0.984375,
       "learning_rate": 8.258081741438394e-07,
-      "loss": 0.0011,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0466,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.14433756470680237,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 361
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 586.7708435058594,
+      "completion_length": 2041.1041870117188,
       "epoch": 0.362,
-      "grad_norm": 1.214355109649776,
-      "kl": 0.03515625,
+      "grad_norm": 4.5128917061627405,
+      "kl": 0.59375,
       "learning_rate": 8.245653237555705e-07,
-      "loss": 0.1163,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.06226281076669693,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.0328,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 362
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 536.7708435058594,
+      "completion_length": 2048.0,
       "epoch": 0.363,
-      "grad_norm": 0.063669726734918,
-      "kl": 0.02801513671875,
+      "grad_norm": 2.213602559064017,
+      "kl": 0.359375,
       "learning_rate": 8.23319127875479e-07,
-      "loss": 0.0011,
-      "reward": 1.0,
+      "loss": 0.0144,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 363
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 555.6041717529297,
+      "completion_length": 2025.0416870117188,
       "epoch": 0.364,
-      "grad_norm": 0.07180792348851471,
-      "kl": 0.03021240234375,
+      "grad_norm": 1.2626343759468237,
+      "kl": 0.2294921875,
       "learning_rate": 8.220696016880687e-07,
-      "loss": 0.0012,
-      "reward": 1.0,
+      "loss": 0.0092,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 364
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 566.625,
+      "completion_length": 2014.7708740234375,
       "epoch": 0.365,
-      "grad_norm": 1.5129871887055757,
-      "kl": 0.031982421875,
+      "grad_norm": 1.323627069076087,
+      "kl": 0.2333984375,
       "learning_rate": 8.208167604184217e-07,
-      "loss": 0.1088,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.07216878235340118,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0094,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 365
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 488.5833435058594,
+      "completion_length": 2048.0,
       "epoch": 0.366,
-      "grad_norm": 0.1253293797973738,
-      "kl": 0.0357666015625,
+      "grad_norm": 1.0838776478162437,
+      "kl": 0.18896484375,
       "learning_rate": 8.195606193320136e-07,
-      "loss": 0.0014,
-      "reward": 1.0,
+      "loss": 0.0076,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 366
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 537.6041870117188,
+      "completion_length": 2024.3333740234375,
       "epoch": 0.367,
-      "grad_norm": 0.09708604189309158,
-      "kl": 0.0328369140625,
+      "grad_norm": 5.032693179785947,
+      "kl": 0.17578125,
       "learning_rate": 8.183011937345271e-07,
-      "loss": 0.0013,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0446,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 367
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 616.9791870117188,
+      "completion_length": 1997.8958740234375,
       "epoch": 0.368,
-      "grad_norm": 1.034557724842816,
-      "kl": 0.0343017578125,
+      "grad_norm": 5.677000796778834,
+      "kl": 0.1591796875,
       "learning_rate": 8.170384989716657e-07,
-      "loss": 0.0395,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0411,
+      "reward": 0.12500000558793545,
+      "reward_std": 0.2825479060411453,
+      "rewards/accuracy_reward": 0.12500000558793545,
       "step": 368
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 541.6041870117188,
+      "completion_length": 1940.8541870117188,
       "epoch": 0.369,
-      "grad_norm": 0.06191459889287263,
-      "kl": 0.0311279296875,
+      "grad_norm": 6.8305107198483475,
+      "kl": 0.1923828125,
       "learning_rate": 8.157725504289664e-07,
-      "loss": 0.0012,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0624,
+      "reward": 0.1666666716337204,
+      "reward_std": 0.3234458789229393,
+      "rewards/accuracy_reward": 0.1666666716337204,
       "step": 369
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 507.9583435058594,
+      "completion_length": 1914.3125610351562,
       "epoch": 0.37,
-      "grad_norm": 1.1542301179234153,
-      "kl": 0.038330078125,
+      "grad_norm": 3.536513490673903,
+      "kl": 0.2490234375,
       "learning_rate": 8.145033635316128e-07,
-      "loss": 0.0442,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0455,
+      "reward": 0.2500000149011612,
+      "reward_std": 0.3979873061180115,
+      "rewards/accuracy_reward": 0.2500000149011612,
       "step": 370
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 540.1250152587891,
+      "completion_length": 1780.9791870117188,
       "epoch": 0.371,
-      "grad_norm": 0.10027568037073406,
-      "kl": 0.03515625,
+      "grad_norm": 9.153488280816177,
+      "kl": 0.3388671875,
       "learning_rate": 8.13230953744247e-07,
-      "loss": 0.0014,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1345,
+      "reward": 0.2083333432674408,
+      "reward_std": 0.4207582473754883,
+      "rewards/accuracy_reward": 0.2083333432674408,
       "step": 371
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 487.6041717529297,
+      "completion_length": 1757.0833740234375,
       "epoch": 0.372,
-      "grad_norm": 0.0874359442204591,
-      "kl": 0.035400390625,
+      "grad_norm": 8.562324197990318,
+      "kl": 0.4384765625,
       "learning_rate": 8.119553365707802e-07,
-      "loss": 0.0014,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1518,
+      "reward": 0.3125,
+      "reward_std": 0.46402883529663086,
+      "rewards/accuracy_reward": 0.3125,
       "step": 372
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 528.9791870117188,
+      "completion_length": 1540.2083740234375,
       "epoch": 0.373,
-      "grad_norm": 0.08394294589398749,
-      "kl": 0.03497314453125,
+      "grad_norm": 11.288500822224922,
+      "kl": 0.5224609375,
       "learning_rate": 8.106765275542053e-07,
-      "loss": 0.0013,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0729,
+      "reward": 0.5833333432674408,
+      "reward_std": 0.5072994828224182,
+      "rewards/accuracy_reward": 0.5833333432674408,
       "step": 373
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 553.1250305175781,
+      "completion_length": 1365.9166870117188,
       "epoch": 0.374,
-      "grad_norm": 0.09439882091925837,
-      "kl": 0.0390625,
+      "grad_norm": 7.326724844915505,
+      "kl": 0.4921875,
       "learning_rate": 8.093945422764069e-07,
-      "loss": 0.0015,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0324,
+      "reward": 0.4375,
+      "reward_std": 0.43824970722198486,
+      "rewards/accuracy_reward": 0.4375,
       "step": 374
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 559.7708740234375,
+      "completion_length": 1311.2083740234375,
       "epoch": 0.375,
-      "grad_norm": 0.05978916201104615,
-      "kl": 0.0321044921875,
+      "grad_norm": 4.2662739479931675,
+      "kl": 0.3125,
       "learning_rate": 8.081093963579707e-07,
-      "loss": 0.0013,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0773,
+      "reward": 0.5208333432674408,
+      "reward_std": 0.498006671667099,
+      "rewards/accuracy_reward": 0.5208333432674408,
       "step": 375
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 547.3958435058594,
+      "completion_length": 1267.1041870117188,
       "epoch": 0.376,
-      "grad_norm": 2.1280601903761647,
-      "kl": 0.03857421875,
+      "grad_norm": 4.728459773940599,
+      "kl": 0.3984375,
       "learning_rate": 8.068211054579943e-07,
-      "loss": 0.2465,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.09021097794175148,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.0487,
+      "reward": 0.6875000298023224,
+      "reward_std": 0.3749151676893234,
+      "rewards/accuracy_reward": 0.6875000298023224,
       "step": 376
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 422.10418701171875,
+      "completion_length": 1294.7500610351562,
       "epoch": 0.377,
-      "grad_norm": 0.08127219256788779,
-      "kl": 0.040283203125,
+      "grad_norm": 9.085999549915973,
+      "kl": 0.5546875,
       "learning_rate": 8.055296852738956e-07,
-      "loss": 0.0016,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0501,
+      "reward": 0.5833333730697632,
+      "reward_std": 0.4778681993484497,
+      "rewards/accuracy_reward": 0.5833333730697632,
       "step": 377
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 545.1875305175781,
+      "completion_length": 1415.0833740234375,
       "epoch": 0.378,
-      "grad_norm": 1.3001640432345112,
-      "kl": 0.03955078125,
+      "grad_norm": 4.6281774284152215,
+      "kl": 0.3896484375,
       "learning_rate": 8.04235151541222e-07,
-      "loss": 0.1207,
-      "reward": 0.984375,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0681,
+      "reward": 0.6666666865348816,
+      "reward_std": 0.3648904263973236,
+      "rewards/accuracy_reward": 0.6666666865348816,
       "step": 378
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 581.6458435058594,
+      "completion_length": 1482.6458740234375,
       "epoch": 0.379,
-      "grad_norm": 0.08477819999323476,
-      "kl": 0.0411376953125,
+      "grad_norm": 10.056997424315483,
+      "kl": 0.4697265625,
       "learning_rate": 8.029375200334587e-07,
-      "loss": 0.0016,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": -0.0002,
+      "reward": 0.6041666865348816,
+      "reward_std": 0.46019135415554047,
+      "rewards/accuracy_reward": 0.6041666865348816,
       "step": 379
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 528.8750305175781,
+      "completion_length": 1632.854248046875,
       "epoch": 0.38,
-      "grad_norm": 0.09753995035880399,
-      "kl": 0.0433349609375,
+      "grad_norm": 10.036223838464284,
+      "kl": 0.490234375,
       "learning_rate": 8.01636806561836e-07,
-      "loss": 0.0017,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0392,
+      "reward": 0.5833333432674408,
+      "reward_std": 0.5111140608787537,
+      "rewards/accuracy_reward": 0.5833333432674408,
       "step": 380
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 507.5833435058594,
+      "completion_length": 1705.791748046875,
       "epoch": 0.381,
-      "grad_norm": 0.999357167905774,
-      "kl": 0.049560546875,
+      "grad_norm": 5.996920133051985,
+      "kl": 0.4228515625,
       "learning_rate": 8.003330269751372e-07,
-      "loss": -0.002,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0488,
+      "reward": 0.75,
+      "reward_std": 0.41310612857341766,
+      "rewards/accuracy_reward": 0.75,
       "step": 381
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 528.8750152587891,
+      "completion_length": 1854.0625610351562,
       "epoch": 0.382,
-      "grad_norm": 0.088362076534267,
-      "kl": 0.0413818359375,
+      "grad_norm": 5.416677061313333,
+      "kl": 0.634765625,
       "learning_rate": 7.990261971595048e-07,
-      "loss": 0.0017,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0637,
+      "reward": 0.3333333432674408,
+      "reward_std": 0.4520890414714813,
+      "rewards/accuracy_reward": 0.3333333432674408,
       "step": 382
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 558.2916870117188,
+      "completion_length": 1771.6458740234375,
       "epoch": 0.383,
-      "grad_norm": 0.8238863782718631,
-      "kl": 0.0421142578125,
+      "grad_norm": 5.808521566964955,
+      "kl": 0.81640625,
       "learning_rate": 7.977163330382479e-07,
-      "loss": -0.0087,
-      "reward": 0.984375,
-      "reward_std": 0.03884884715080261,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.065,
+      "reward": 0.4166666716337204,
+      "reward_std": 0.49544864892959595,
+      "rewards/accuracy_reward": 0.4166666716337204,
       "step": 383
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 502.7083435058594,
+      "completion_length": 1797.3333740234375,
       "epoch": 0.384,
-      "grad_norm": 0.09055795872538838,
-      "kl": 0.0428466796875,
+      "grad_norm": 8.280633894732846,
+      "kl": 1.056640625,
       "learning_rate": 7.964034505716476e-07,
-      "loss": 0.0017,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0333,
+      "reward": 0.4583333432674408,
+      "reward_std": 0.4213048070669174,
+      "rewards/accuracy_reward": 0.4583333432674408,
       "step": 384
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 589.5625305175781,
+      "completion_length": 1850.8541870117188,
       "epoch": 0.385,
-      "grad_norm": 1.1676528135282047,
-      "kl": 0.0408935546875,
+      "grad_norm": 5.732570494260635,
+      "kl": 1.150390625,
       "learning_rate": 7.950875657567621e-07,
-      "loss": 0.0309,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0841,
+      "reward": 0.2500000149011612,
+      "reward_std": 0.4314185380935669,
+      "rewards/accuracy_reward": 0.2500000149011612,
       "step": 385
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 556.75,
+      "completion_length": 1851.3333740234375,
       "epoch": 0.386,
-      "grad_norm": 1.1667991725242814,
-      "kl": 0.04248046875,
+      "grad_norm": 4.159878613517792,
+      "kl": 0.962890625,
       "learning_rate": 7.93768694627233e-07,
-      "loss": 0.061,
-      "reward": 0.984375,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0119,
+      "reward": 0.291666679084301,
+      "reward_std": 0.29193708300590515,
+      "rewards/accuracy_reward": 0.291666679084301,
       "step": 386
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 501.25001525878906,
+      "completion_length": 1812.75,
       "epoch": 0.387,
-      "grad_norm": 1.552193185021972,
-      "kl": 0.0416259765625,
+      "grad_norm": 4.061483859344053,
+      "kl": 0.919921875,
       "learning_rate": 7.924468532530883e-07,
-      "loss": 0.149,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0426,
+      "reward": 0.12500000558793545,
+      "reward_std": 0.26742906868457794,
+      "rewards/accuracy_reward": 0.12500000558793545,
       "step": 387
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 564.6041870117188,
+      "completion_length": 1854.4375610351562,
       "epoch": 0.388,
-      "grad_norm": 2.451979385765991,
-      "kl": 0.04248046875,
+      "grad_norm": 6.932039336200126,
+      "kl": 0.921875,
       "learning_rate": 7.911220577405484e-07,
-      "loss": 0.2295,
-      "reward": 0.953125,
-      "reward_std": 0.16237976029515266,
-      "rewards/tag_count_reward": 0.953125,
+      "loss": 0.0339,
+      "reward": 0.3333333432674408,
+      "reward_std": 0.4778681844472885,
+      "rewards/accuracy_reward": 0.3333333432674408,
       "step": 388
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 474.9166717529297,
+      "completion_length": 1738.6666870117188,
       "epoch": 0.389,
-      "grad_norm": 1.4403584875210205,
-      "kl": 0.0487060546875,
+      "grad_norm": 5.487440860172792,
+      "kl": 0.91015625,
       "learning_rate": 7.897943242318285e-07,
-      "loss": 0.0261,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": -0.0061,
+      "reward": 0.375,
+      "reward_std": 0.4778681993484497,
+      "rewards/accuracy_reward": 0.375,
       "step": 389
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 568.9791870117188,
+      "completion_length": 1822.1041870117188,
       "epoch": 0.39,
-      "grad_norm": 0.09658274855884183,
-      "kl": 0.0390625,
+      "grad_norm": 6.325487028580445,
+      "kl": 0.794921875,
       "learning_rate": 7.884636689049422e-07,
-      "loss": 0.0016,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0879,
+      "reward": 0.5833333432674408,
+      "reward_std": 0.4213048070669174,
+      "rewards/accuracy_reward": 0.5833333432674408,
       "step": 390
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 503.47918701171875,
+      "completion_length": 1587.1458740234375,
       "epoch": 0.391,
-      "grad_norm": 0.10687541663572057,
-      "kl": 0.045654296875,
+      "grad_norm": 14.647138502761237,
+      "kl": 1.59765625,
       "learning_rate": 7.871301079735049e-07,
-      "loss": 0.0018,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": -0.0591,
+      "reward": 0.4791666716337204,
+      "reward_std": 0.47795717418193817,
+      "rewards/accuracy_reward": 0.4791666716337204,
       "step": 391
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 554.7708435058594,
+      "completion_length": 1784.5208740234375,
       "epoch": 0.392,
-      "grad_norm": 2.133786999649764,
-      "kl": 0.035888671875,
+      "grad_norm": 5.026391303039424,
+      "kl": 1.103515625,
       "learning_rate": 7.857936576865356e-07,
-      "loss": 0.1066,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.07845467701554298,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.0375,
+      "reward": 0.4583333544433117,
+      "reward_std": 0.33896228671073914,
+      "rewards/accuracy_reward": 0.4583333544433117,
       "step": 392
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 487.9583435058594,
+      "completion_length": 1708.7916870117188,
       "epoch": 0.393,
-      "grad_norm": 1.6908693537415456,
-      "kl": 0.0416259765625,
+      "grad_norm": 9.993145253136335,
+      "kl": 1.35546875,
       "learning_rate": 7.844543343282595e-07,
-      "loss": 0.088,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": -0.0212,
+      "reward": 0.6458333432674408,
+      "reward_std": 0.4678434729576111,
+      "rewards/accuracy_reward": 0.6458333432674408,
       "step": 393
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 486.0208435058594,
+      "completion_length": 1792.5833740234375,
       "epoch": 0.394,
-      "grad_norm": 0.10411848264951166,
-      "kl": 0.046630859375,
+      "grad_norm": 7.927088260682654,
+      "kl": 1.00390625,
       "learning_rate": 7.831121542179086e-07,
-      "loss": 0.0019,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": -0.0212,
+      "reward": 0.8333333730697632,
+      "reward_std": 0.3083270490169525,
+      "rewards/accuracy_reward": 0.8333333730697632,
       "step": 394
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 484.8541717529297,
+      "completion_length": 1818.1666870117188,
       "epoch": 0.395,
-      "grad_norm": 0.14391019717031175,
-      "kl": 0.0447998046875,
+      "grad_norm": 4.737258512977974,
+      "kl": 1.4609375,
       "learning_rate": 7.817671337095244e-07,
-      "loss": 0.0018,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.071,
+      "reward": 0.2500000149011612,
+      "reward_std": 0.3979873061180115,
+      "rewards/accuracy_reward": 0.2500000149011612,
       "step": 395
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 471.0833435058594,
+      "completion_length": 1819.8541870117188,
       "epoch": 0.396,
-      "grad_norm": 0.12688373205559,
-      "kl": 0.0457763671875,
+      "grad_norm": 6.306404855697497,
+      "kl": 1.4453125,
       "learning_rate": 7.804192891917571e-07,
-      "loss": 0.0018,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": -0.02,
+      "reward": 0.3125,
+      "reward_std": 0.44452594220638275,
+      "rewards/accuracy_reward": 0.3125,
       "step": 396
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 528.9375,
+      "completion_length": 1860.6250610351562,
       "epoch": 0.397,
-      "grad_norm": 0.13324985505443265,
-      "kl": 0.0406494140625,
+      "grad_norm": 4.340406391811811,
+      "kl": 1.021484375,
       "learning_rate": 7.79068637087667e-07,
-      "loss": 0.0016,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0843,
+      "reward": 0.3125,
+      "reward_std": 0.4269454777240753,
+      "rewards/accuracy_reward": 0.3125,
       "step": 397
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 519.5208435058594,
+      "completion_length": 1739.9166870117188,
       "epoch": 0.398,
-      "grad_norm": 1.0531523617961214,
-      "kl": 0.05224609375,
+      "grad_norm": 5.025838221279123,
+      "kl": 1.7421875,
       "learning_rate": 7.777151938545235e-07,
-      "loss": 0.0243,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0173,
+      "reward": 0.5416666865348816,
+      "reward_std": 0.49544869363307953,
+      "rewards/accuracy_reward": 0.5416666865348816,
       "step": 398
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 447.6666717529297,
+      "completion_length": 1677.3125,
       "epoch": 0.399,
-      "grad_norm": 0.12314730150608288,
-      "kl": 0.0491943359375,
+      "grad_norm": 4.609305414192944,
+      "kl": 1.92578125,
       "learning_rate": 7.763589759836058e-07,
-      "loss": 0.002,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0922,
+      "reward": 0.2916666716337204,
+      "reward_std": 0.46793243288993835,
+      "rewards/accuracy_reward": 0.2916666716337204,
       "step": 399
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 535.2916870117188,
+      "completion_length": 1729.8958740234375,
       "epoch": 0.4,
-      "grad_norm": 0.13208913700559177,
-      "kl": 0.0465087890625,
+      "grad_norm": 7.322509625138369,
+      "kl": 1.69921875,
       "learning_rate": 7.75e-07,
-      "loss": 0.0019,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": -0.0017,
+      "reward": 0.3125000149011612,
+      "reward_std": 0.45391513407230377,
+      "rewards/accuracy_reward": 0.3125000149011612,
       "step": 400
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 475.7916717529297,
+      "completion_length": 1794.1250610351562,
       "epoch": 0.401,
-      "grad_norm": 0.11631057614821307,
-      "kl": 0.050537109375,
+      "grad_norm": 6.3633323886265485,
+      "kl": 2.1328125,
       "learning_rate": 7.736382824623999e-07,
-      "loss": 0.002,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0822,
+      "reward": 0.25,
+      "reward_std": 0.4408077001571655,
+      "rewards/accuracy_reward": 0.25,
       "step": 401
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 535.6666870117188,
+      "completion_length": 1722.1875610351562,
       "epoch": 0.402,
-      "grad_norm": 1.5757509818707738,
-      "kl": 0.0526123046875,
+      "grad_norm": 9.593685306614804,
+      "kl": 1.5546875,
       "learning_rate": 7.72273839962904e-07,
-      "loss": 0.0746,
-      "reward": 0.9739583730697632,
-      "reward_std": 0.09021097794175148,
-      "rewards/tag_count_reward": 0.9739583730697632,
+      "loss": 0.1104,
+      "reward": 0.2291666679084301,
+      "reward_std": 0.3409373462200165,
+      "rewards/accuracy_reward": 0.2291666679084301,
       "step": 402
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 570.4166870117188,
+      "completion_length": 1503.1041870117188,
       "epoch": 0.403,
-      "grad_norm": 1.1336688099849115,
-      "kl": 0.048828125,
+      "grad_norm": 14.679314451353129,
+      "kl": 1.9375,
       "learning_rate": 7.709066891268133e-07,
-      "loss": 0.0292,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.1852,
+      "reward": 0.2916666716337204,
+      "reward_std": 0.42877157032489777,
+      "rewards/accuracy_reward": 0.2916666716337204,
       "step": 403
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 520.75,
+      "completion_length": 1823.291748046875,
       "epoch": 0.404,
-      "grad_norm": 0.10134365577177148,
-      "kl": 0.046630859375,
+      "grad_norm": 10.173189728059626,
+      "kl": 1.52734375,
       "learning_rate": 7.695368466124296e-07,
-      "loss": 0.0019,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0854,
+      "reward": 0.1041666679084301,
+      "reward_std": 0.26679351925849915,
+      "rewards/accuracy_reward": 0.1041666679084301,
       "step": 404
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 519.0416717529297,
+      "completion_length": 1851.9375610351562,
       "epoch": 0.405,
-      "grad_norm": 0.09498459340343707,
-      "kl": 0.0439453125,
+      "grad_norm": 7.210912415755655,
+      "kl": 1.19921875,
       "learning_rate": 7.681643291108517e-07,
-      "loss": 0.0018,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1206,
+      "reward": 0.10416666977107525,
+      "reward_std": 0.19526028633117676,
+      "rewards/accuracy_reward": 0.10416666977107525,
       "step": 405
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 509.3125305175781,
+      "completion_length": 1870.666748046875,
       "epoch": 0.406,
-      "grad_norm": 0.09249419254408961,
-      "kl": 0.041259765625,
+      "grad_norm": 8.768417979133082,
+      "kl": 1.2890625,
       "learning_rate": 7.667891533457718e-07,
-      "loss": 0.0016,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1072,
+      "reward": 0.1041666716337204,
+      "reward_std": 0.19526028633117676,
+      "rewards/accuracy_reward": 0.1041666716337204,
       "step": 406
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 500.3125305175781,
+      "completion_length": 2026.7708740234375,
       "epoch": 0.407,
-      "grad_norm": 0.1014194854177954,
-      "kl": 0.0462646484375,
+      "grad_norm": 5.571215140704089,
+      "kl": 1.14453125,
       "learning_rate": 7.654113360732732e-07,
-      "loss": 0.0019,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0788,
+      "reward": 0.06250000186264515,
+      "reward_std": 0.21650634706020355,
+      "rewards/accuracy_reward": 0.06250000186264515,
       "step": 407
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 531.7500152587891,
+      "completion_length": 2011.6458740234375,
       "epoch": 0.408,
-      "grad_norm": 0.0850335798970897,
-      "kl": 0.0423583984375,
+      "grad_norm": 0.73419253872998,
+      "kl": 1.044921875,
       "learning_rate": 7.640308940816239e-07,
-      "loss": 0.0017,
-      "reward": 1.0,
+      "loss": 0.0424,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 408
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 498.7291717529297,
+      "completion_length": 2048.0,
       "epoch": 0.409,
-      "grad_norm": 0.09839539734571516,
-      "kl": 0.047119140625,
+      "grad_norm": 0.5134992790863855,
+      "kl": 1.08984375,
       "learning_rate": 7.626478441910744e-07,
-      "loss": 0.0019,
-      "reward": 1.0,
+      "loss": 0.0436,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 409
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 525.7083435058594,
+      "completion_length": 2033.4583740234375,
       "epoch": 0.41,
-      "grad_norm": 0.09239514842845858,
-      "kl": 0.0416259765625,
+      "grad_norm": 0.7673559838779002,
+      "kl": 1.06640625,
       "learning_rate": 7.612622032536507e-07,
-      "loss": 0.0017,
-      "reward": 1.0,
+      "loss": 0.0426,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 410
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 519.5833435058594,
+      "completion_length": 2044.625,
       "epoch": 0.411,
-      "grad_norm": 0.09033270466242478,
-      "kl": 0.0458984375,
+      "grad_norm": 1.2563822980972001,
+      "kl": 0.951171875,
       "learning_rate": 7.59873988152951e-07,
-      "loss": 0.0018,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0431,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 411
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 452.8333435058594,
+      "completion_length": 2048.0,
       "epoch": 0.412,
-      "grad_norm": 0.0849946940770293,
-      "kl": 0.046142578125,
+      "grad_norm": 0.4990850511955262,
+      "kl": 0.921875,
       "learning_rate": 7.584832158039378e-07,
-      "loss": 0.0018,
-      "reward": 1.0,
+      "loss": 0.0369,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 412
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 479.5833435058594,
+      "completion_length": 2018.3125,
       "epoch": 0.413,
-      "grad_norm": 0.09480109650231232,
-      "kl": 0.051025390625,
+      "grad_norm": 0.5216315176531795,
+      "kl": 0.814453125,
       "learning_rate": 7.570899031527332e-07,
-      "loss": 0.002,
-      "reward": 1.0,
+      "loss": 0.033,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 413
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 512.3958435058594,
+      "completion_length": 2048.0,
       "epoch": 0.414,
-      "grad_norm": 0.08812828615766397,
-      "kl": 0.045654296875,
+      "grad_norm": 0.6210781963395747,
+      "kl": 0.5703125,
       "learning_rate": 7.556940671764124e-07,
-      "loss": 0.0019,
-      "reward": 1.0,
+      "loss": 0.0228,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 414
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 528.8541870117188,
+      "completion_length": 2045.4583740234375,
       "epoch": 0.415,
-      "grad_norm": 0.08811692768903577,
-      "kl": 0.0443115234375,
+      "grad_norm": 1.7766404519898162,
+      "kl": 0.4697265625,
       "learning_rate": 7.54295724882796e-07,
-      "loss": 0.0018,
-      "reward": 1.0,
+      "loss": 0.0188,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 415
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 520.2500305175781,
+      "completion_length": 2048.0,
       "epoch": 0.416,
-      "grad_norm": 0.07578652447775983,
-      "kl": 0.0396728515625,
+      "grad_norm": 1.1616559759230074,
+      "kl": 0.4267578125,
       "learning_rate": 7.528948933102438e-07,
-      "loss": 0.0016,
-      "reward": 1.0,
+      "loss": 0.0171,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 416
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 561.4375305175781,
+      "completion_length": 2010.625,
       "epoch": 0.417,
-      "grad_norm": 0.07895167806367016,
-      "kl": 0.0433349609375,
+      "grad_norm": 0.4610497179696256,
+      "kl": 0.5107421875,
       "learning_rate": 7.514915895274463e-07,
-      "loss": 0.0017,
-      "reward": 1.0,
+      "loss": 0.0208,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 417
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 485.12501525878906,
+      "completion_length": 2048.0,
       "epoch": 0.418,
-      "grad_norm": 0.08518158009411357,
-      "kl": 0.046142578125,
+      "grad_norm": 0.7805525873376801,
+      "kl": 0.61328125,
       "learning_rate": 7.500858306332172e-07,
-      "loss": 0.0019,
-      "reward": 1.0,
+      "loss": 0.0246,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 418
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 508.97918701171875,
+      "completion_length": 2048.0,
       "epoch": 0.419,
-      "grad_norm": 1.0053918619854647,
-      "kl": 0.0478515625,
+      "grad_norm": 2.8888656890071793,
+      "kl": 0.623046875,
       "learning_rate": 7.486776337562853e-07,
-      "loss": 0.0063,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.025,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 419
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 526.1666870117188,
+      "completion_length": 2048.0,
       "epoch": 0.42,
-      "grad_norm": 1.0912738995624687,
-      "kl": 0.044677734375,
+      "grad_norm": 0.6628463607523004,
+      "kl": 0.4521484375,
       "learning_rate": 7.472670160550848e-07,
-      "loss": 0.038,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0181,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 420
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 515.4791870117188,
+      "completion_length": 2048.0,
       "epoch": 0.421,
-      "grad_norm": 0.10769262285700905,
-      "kl": 0.0389404296875,
+      "grad_norm": 0.6202863156484869,
+      "kl": 0.38671875,
       "learning_rate": 7.458539947175473e-07,
-      "loss": 0.0016,
-      "reward": 1.0,
+      "loss": 0.0154,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 421
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 542.8333435058594,
+      "completion_length": 2048.0,
       "epoch": 0.422,
-      "grad_norm": 0.124859076624025,
-      "kl": 0.0406494140625,
+      "grad_norm": 0.8618649510766347,
+      "kl": 0.4072265625,
       "learning_rate": 7.444385869608921e-07,
-      "loss": 0.0016,
-      "reward": 1.0,
+      "loss": 0.0163,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 422
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 557.9583435058594,
+      "completion_length": 2020.625,
       "epoch": 0.423,
-      "grad_norm": 0.06978524202047731,
-      "kl": 0.037841796875,
+      "grad_norm": 0.5814051104259365,
+      "kl": 0.3681640625,
       "learning_rate": 7.430208100314156e-07,
-      "loss": 0.0015,
-      "reward": 1.0,
+      "loss": 0.0149,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 423
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 510.91668701171875,
+      "completion_length": 2048.0,
       "epoch": 0.424,
-      "grad_norm": 0.9837927706804174,
-      "kl": 0.0379638671875,
+      "grad_norm": 0.3317922423103524,
+      "kl": 0.39453125,
       "learning_rate": 7.416006812042827e-07,
-      "loss": 0.007,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0158,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 424
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 540.6666870117188,
+      "completion_length": 2048.0,
       "epoch": 0.425,
-      "grad_norm": 2.9312181195112146,
-      "kl": 0.045654296875,
+      "grad_norm": 0.6260973894330396,
+      "kl": 0.4404296875,
       "learning_rate": 7.401782177833147e-07,
-      "loss": 0.213,
-      "reward": 0.9687500298023224,
-      "reward_std": 0.10825317353010178,
-      "rewards/tag_count_reward": 0.9687500298023224,
+      "loss": 0.0176,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 425
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 527.2708435058594,
+      "completion_length": 2048.0,
       "epoch": 0.426,
-      "grad_norm": 0.09640325328962256,
-      "kl": 0.0430908203125,
+      "grad_norm": 0.599756203951725,
+      "kl": 0.3857421875,
       "learning_rate": 7.387534371007797e-07,
-      "loss": 0.0017,
-      "reward": 1.0,
+      "loss": 0.0154,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 426
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 519.8541870117188,
+      "completion_length": 2048.0,
       "epoch": 0.427,
-      "grad_norm": 0.07472945475941242,
-      "kl": 0.04296875,
+      "grad_norm": 0.6574068588195098,
+      "kl": 0.3369140625,
       "learning_rate": 7.373263565171805e-07,
-      "loss": 0.0017,
-      "reward": 1.0,
+      "loss": 0.0135,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 427
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 480.4583435058594,
+      "completion_length": 2025.2083740234375,
       "epoch": 0.428,
-      "grad_norm": 0.1283159299656476,
-      "kl": 0.046875,
+      "grad_norm": 2.29129301623511,
+      "kl": 0.2900390625,
       "learning_rate": 7.358969934210438e-07,
-      "loss": 0.0018,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0378,
+      "reward": 0.0625,
+      "reward_std": 0.16948114335536957,
+      "rewards/accuracy_reward": 0.0625,
       "step": 428
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 527.8125305175781,
+      "completion_length": 2048.0,
       "epoch": 0.429,
-      "grad_norm": 1.2109285264735294,
-      "kl": 0.0404052734375,
+      "grad_norm": 1.6059902502930246,
+      "kl": 0.3662109375,
       "learning_rate": 7.344653652287077e-07,
-      "loss": 0.0558,
-      "reward": 0.984375,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0146,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 429
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 572.6875305175781,
+      "completion_length": 2037.3333740234375,
       "epoch": 0.43,
-      "grad_norm": 0.12680186205193347,
-      "kl": 0.043701171875,
+      "grad_norm": 3.4436184353174717,
+      "kl": 0.33203125,
       "learning_rate": 7.330314893841101e-07,
-      "loss": 0.0018,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0295,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.14433756470680237,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 430
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 502.0000305175781,
+      "completion_length": 1971.3125610351562,
       "epoch": 0.431,
-      "grad_norm": 0.08913699535865231,
-      "kl": 0.0457763671875,
+      "grad_norm": 6.670962406275426,
+      "kl": 0.294921875,
       "learning_rate": 7.315953833585755e-07,
-      "loss": 0.0018,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0564,
+      "reward": 0.0833333358168602,
+      "reward_std": 0.24164992570877075,
+      "rewards/accuracy_reward": 0.0833333358168602,
       "step": 431
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 652.8333435058594,
+      "completion_length": 1845.2083740234375,
       "epoch": 0.432,
-      "grad_norm": 1.1575484907375526,
-      "kl": 0.0435791015625,
+      "grad_norm": 9.465275682108377,
+      "kl": 0.25048828125,
       "learning_rate": 7.301570646506027e-07,
-      "loss": 0.0294,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0944,
+      "reward": 0.2083333432674408,
+      "reward_std": 0.39552566409111023,
+      "rewards/accuracy_reward": 0.2083333432674408,
       "step": 432
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 557.9791870117188,
+      "completion_length": 1629.9375610351562,
       "epoch": 0.433,
-      "grad_norm": 1.2239767941332247,
-      "kl": 0.0479736328125,
+      "grad_norm": 21.46667610929408,
+      "kl": 0.3828125,
       "learning_rate": 7.287165507856512e-07,
-      "loss": 0.0179,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.1765,
+      "reward": 0.4375000149011612,
+      "reward_std": 0.4936225861310959,
+      "rewards/accuracy_reward": 0.4375000149011612,
       "step": 433
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 501.93751525878906,
+      "completion_length": 1279.5625610351562,
       "epoch": 0.434,
-      "grad_norm": 1.6103755307461893,
-      "kl": 0.0455322265625,
+      "grad_norm": 18.635871984112118,
+      "kl": 0.673828125,
       "learning_rate": 7.27273859315928e-07,
-      "loss": 0.0154,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.09021097794175148,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.1711,
+      "reward": 0.7291666865348816,
+      "reward_std": 0.452178031206131,
+      "rewards/accuracy_reward": 0.7291666865348816,
       "step": 434
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 547.8541870117188,
+      "completion_length": 1002.0833740234375,
       "epoch": 0.435,
-      "grad_norm": 1.3375839613733431,
-      "kl": 0.0462646484375,
+      "grad_norm": 12.003672953951181,
+      "kl": 0.712890625,
       "learning_rate": 7.258290078201731e-07,
-      "loss": 0.0108,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0323,
+      "reward": 0.458333358168602,
+      "reward_std": 0.3979873061180115,
+      "rewards/accuracy_reward": 0.458333358168602,
       "step": 435
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 543.0625305175781,
+      "completion_length": 930.0416870117188,
       "epoch": 0.436,
-      "grad_norm": 1.9877666897516044,
-      "kl": 0.049072265625,
+      "grad_norm": 21.91073258153116,
+      "kl": 0.75,
       "learning_rate": 7.243820139034464e-07,
-      "loss": 0.1998,
-      "reward": 0.96875,
-      "reward_std": 0.07298427820205688,
-      "rewards/tag_count_reward": 0.96875,
+      "loss": -0.0079,
+      "reward": 0.7708333432674408,
+      "reward_std": 0.4112800806760788,
+      "rewards/accuracy_reward": 0.7708333432674408,
       "step": 436
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 614.6875305175781,
+      "completion_length": 1108.6458740234375,
       "epoch": 0.437,
-      "grad_norm": 3.393227912741051,
-      "kl": 0.0947265625,
+      "grad_norm": 25.304512135344478,
+      "kl": 0.798828125,
       "learning_rate": 7.229328951969115e-07,
-      "loss": 0.1849,
-      "reward": 0.9531250298023224,
-      "reward_std": 0.14568757265806198,
-      "rewards/tag_count_reward": 0.9531250298023224,
+      "loss": 0.0185,
+      "reward": 0.6875000298023224,
+      "reward_std": 0.3749151825904846,
+      "rewards/accuracy_reward": 0.6875000298023224,
       "step": 437
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 605.0625,
+      "completion_length": 1267.8125,
       "epoch": 0.438,
-      "grad_norm": 2.9704332189234335,
-      "kl": 0.0576171875,
+      "grad_norm": 27.52378691853374,
+      "kl": 0.9296875,
       "learning_rate": 7.214816693576234e-07,
-      "loss": 0.3069,
-      "reward": 0.953125,
-      "reward_std": 0.14568756893277168,
-      "rewards/tag_count_reward": 0.953125,
+      "loss": 0.0473,
+      "reward": 0.6875,
+      "reward_std": 0.4527246206998825,
+      "rewards/accuracy_reward": 0.6875,
       "step": 438
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 534.125,
+      "completion_length": 1342.604248046875,
       "epoch": 0.439,
-      "grad_norm": 1.2044060882375394,
-      "kl": 0.058837890625,
+      "grad_norm": 7.639508800331991,
+      "kl": 0.4228515625,
       "learning_rate": 7.200283540683102e-07,
-      "loss": 0.073,
-      "reward": 0.984375,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0796,
+      "reward": 0.5208333432674408,
+      "reward_std": 0.46229176223278046,
+      "rewards/accuracy_reward": 0.5208333432674408,
       "step": 439
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 648.7500305175781,
+      "completion_length": 1604.4583740234375,
       "epoch": 0.44,
-      "grad_norm": 0.19633187817219797,
-      "kl": 0.079833984375,
+      "grad_norm": 12.778609106988583,
+      "kl": 0.2998046875,
       "learning_rate": 7.185729670371604e-07,
-      "loss": 0.0034,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": -0.0161,
+      "reward": 0.5625,
+      "reward_std": 0.498006671667099,
+      "rewards/accuracy_reward": 0.5625,
       "step": 440
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 734.0416870117188,
+      "completion_length": 1784.4791870117188,
       "epoch": 0.441,
-      "grad_norm": 3.2067961154241917,
-      "kl": 0.078369140625,
+      "grad_norm": 4.5200729050884165,
+      "kl": 0.1884765625,
       "learning_rate": 7.171155259976057e-07,
-      "loss": 0.3304,
-      "reward": 0.9635416865348816,
-      "reward_std": 0.12629536911845207,
-      "rewards/tag_count_reward": 0.9635416865348816,
+      "loss": -0.0029,
+      "reward": 0.4166666865348816,
+      "reward_std": 0.4056393951177597,
+      "rewards/accuracy_reward": 0.4166666865348816,
       "step": 441
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 626.3750305175781,
+      "completion_length": 1965.1666870117188,
       "epoch": 0.442,
-      "grad_norm": 2.5378060357093317,
-      "kl": 0.090087890625,
+      "grad_norm": 5.034803772591838,
+      "kl": 0.189453125,
       "learning_rate": 7.156560487081051e-07,
-      "loss": 0.2675,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.07216878235340118,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0116,
+      "reward": 0.2291666716337204,
+      "reward_std": 0.3961612433195114,
+      "rewards/accuracy_reward": 0.2291666716337204,
       "step": 442
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 566.5625152587891,
+      "completion_length": 2001.4375610351562,
       "epoch": 0.443,
-      "grad_norm": 2.7647180072565956,
-      "kl": 0.094482421875,
+      "grad_norm": 7.097908508997813,
+      "kl": 0.19921875,
       "learning_rate": 7.141945529519288e-07,
-      "loss": 0.2807,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.07216878235340118,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0283,
+      "reward": 0.2500000149011612,
+      "reward_std": 0.4213048070669174,
+      "rewards/accuracy_reward": 0.2500000149011612,
       "step": 443
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 625.8750305175781,
+      "completion_length": 1997.0,
       "epoch": 0.444,
-      "grad_norm": 2.723551464343659,
-      "kl": 0.10546875,
+      "grad_norm": 3.7135193848589445,
+      "kl": 0.193359375,
       "learning_rate": 7.127310565369415e-07,
-      "loss": 0.2434,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.07216878235340118,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0305,
+      "reward": 0.125,
+      "reward_std": 0.28254789859056473,
+      "rewards/accuracy_reward": 0.125,
       "step": 444
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 662.7291870117188,
+      "completion_length": 1959.5208740234375,
       "epoch": 0.445,
-      "grad_norm": 2.9676862937048174,
-      "kl": 0.13671875,
+      "grad_norm": 4.364674631503786,
+      "kl": 0.20263671875,
       "learning_rate": 7.11265577295385e-07,
-      "loss": 0.239,
-      "reward": 0.9739583730697632,
-      "reward_std": 0.07845467701554298,
-      "rewards/tag_count_reward": 0.9739583730697632,
+      "loss": 0.0336,
+      "reward": 0.3125,
+      "reward_std": 0.46402883529663086,
+      "rewards/accuracy_reward": 0.3125,
       "step": 445
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 831.4166870117188,
+      "completion_length": 1916.0000610351562,
       "epoch": 0.446,
-      "grad_norm": 1.0131205931227645,
-      "kl": 0.18359375,
+      "grad_norm": 5.718720854624421,
+      "kl": 0.2099609375,
       "learning_rate": 7.097981330836616e-07,
-      "loss": 0.0745,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.024328090250492096,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0321,
+      "reward": 0.3750000149011612,
+      "reward_std": 0.4314185529947281,
+      "rewards/accuracy_reward": 0.3750000149011612,
       "step": 446
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 761.2500305175781,
+      "completion_length": 1822.6250610351562,
       "epoch": 0.447,
-      "grad_norm": 3.0412955207595678,
-      "kl": 0.2373046875,
+      "grad_norm": 7.94819198831549,
+      "kl": 0.2490234375,
       "learning_rate": 7.083287417821157e-07,
-      "loss": 0.245,
-      "reward": 0.9687500298023224,
-      "reward_std": 0.08474057167768478,
-      "rewards/tag_count_reward": 0.9687500298023224,
+      "loss": 0.0104,
+      "reward": 0.5000000298023224,
+      "reward_std": 0.47405362129211426,
+      "rewards/accuracy_reward": 0.5000000298023224,
       "step": 447
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 750.4166870117188,
+      "completion_length": 1664.916748046875,
       "epoch": 0.448,
-      "grad_norm": 1.7160871685898567,
-      "kl": 0.224609375,
+      "grad_norm": 10.544824212937625,
+      "kl": 0.20068359375,
       "learning_rate": 7.068574212948169e-07,
-      "loss": 0.1315,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0234,
+      "reward": 0.6666666865348816,
+      "reward_std": 0.4470839649438858,
+      "rewards/accuracy_reward": 0.6666666865348816,
       "step": 448
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 891.5208740234375,
+      "completion_length": 1672.2083740234375,
       "epoch": 0.449,
-      "grad_norm": 6.204046081883707,
-      "kl": 0.3203125,
+      "grad_norm": 8.893009232109582,
+      "kl": 0.2177734375,
       "learning_rate": 7.053841895493406e-07,
-      "loss": 0.4129,
-      "reward": 0.9218750298023224,
-      "reward_std": 0.17826225608587265,
-      "rewards/tag_count_reward": 0.9218750298023224,
+      "loss": -0.0362,
+      "reward": 0.7083333432674408,
+      "reward_std": 0.4482744634151459,
+      "rewards/accuracy_reward": 0.7083333432674408,
       "step": 449
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 881.5833740234375,
+      "completion_length": 1622.7916870117188,
       "epoch": 0.45,
-      "grad_norm": 3.2434161686257363,
-      "kl": 0.3544921875,
+      "grad_norm": 7.060436269940362,
+      "kl": 0.24951171875,
       "learning_rate": 7.039090644965509e-07,
-      "loss": 0.1983,
-      "reward": 0.953125,
-      "reward_std": 0.09235543012619019,
-      "rewards/tag_count_reward": 0.953125,
+      "loss": -0.0008,
+      "reward": 0.520833358168602,
+      "reward_std": 0.3296331316232681,
+      "rewards/accuracy_reward": 0.520833358168602,
       "step": 450
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1146.0416870117188,
+      "completion_length": 1463.979248046875,
       "epoch": 0.451,
-      "grad_norm": 4.693128676564285,
-      "kl": 0.40234375,
+      "grad_norm": 3.698230709803641,
+      "kl": 0.24365234375,
       "learning_rate": 7.024320641103811e-07,
-      "loss": 0.3054,
-      "reward": 0.9010416865348816,
-      "reward_std": 0.18217922747135162,
-      "rewards/tag_count_reward": 0.9010416865348816,
+      "loss": 0.0052,
+      "reward": 0.5833333432674408,
+      "reward_std": 0.3880225867033005,
+      "rewards/accuracy_reward": 0.5833333432674408,
       "step": 451
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1136.0000610351562,
+      "completion_length": 1502.229248046875,
       "epoch": 0.452,
-      "grad_norm": 8.939444249342907,
-      "kl": 0.49609375,
+      "grad_norm": 5.708275709709953,
+      "kl": 0.22314453125,
       "learning_rate": 7.009532063876148e-07,
-      "loss": 0.4582,
-      "reward": 0.8697916865348816,
-      "reward_std": 0.21116822957992554,
-      "rewards/tag_count_reward": 0.8697916865348816,
+      "loss": -0.0298,
+      "reward": 0.4166666716337204,
+      "reward_std": 0.487981915473938,
+      "rewards/accuracy_reward": 0.4166666716337204,
       "step": 452
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1136.5625,
+      "completion_length": 1649.6250610351562,
       "epoch": 0.453,
-      "grad_norm": 9.611421819429221,
-      "kl": 0.66015625,
+      "grad_norm": 6.745627698285408,
+      "kl": 0.3447265625,
       "learning_rate": 6.994725093476664e-07,
-      "loss": 0.4156,
-      "reward": 0.8645833432674408,
-      "reward_std": 0.22167008370161057,
-      "rewards/tag_count_reward": 0.8645833432674408,
+      "loss": 0.0594,
+      "reward": 0.2916666716337204,
+      "reward_std": 0.3798602670431137,
+      "rewards/accuracy_reward": 0.2916666716337204,
       "step": 453
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1111.6875,
+      "completion_length": 1812.8333740234375,
       "epoch": 0.454,
-      "grad_norm": 10.651125312059772,
-      "kl": 0.732421875,
+      "grad_norm": 6.304630381392028,
+      "kl": 0.3125,
       "learning_rate": 6.979899910323624e-07,
-      "loss": 0.4229,
-      "reward": 0.8697916865348816,
-      "reward_std": 0.219189815223217,
-      "rewards/tag_count_reward": 0.8697916865348816,
+      "loss": 0.0799,
+      "reward": 0.3125000149011612,
+      "reward_std": 0.44891005754470825,
+      "rewards/accuracy_reward": 0.3125000149011612,
       "step": 454
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 967.6042175292969,
+      "completion_length": 1787.5208740234375,
       "epoch": 0.455,
-      "grad_norm": 10.978588111511952,
-      "kl": 0.96484375,
+      "grad_norm": 7.33552455294654,
+      "kl": 0.3447265625,
       "learning_rate": 6.965056695057204e-07,
-      "loss": 0.4329,
-      "reward": 0.9166666865348816,
-      "reward_std": 0.17608554661273956,
-      "rewards/tag_count_reward": 0.9166666865348816,
+      "loss": -0.0061,
+      "reward": 0.2916666865348816,
+      "reward_std": 0.43888527154922485,
+      "rewards/accuracy_reward": 0.2916666865348816,
       "step": 455
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1440.0000610351562,
+      "completion_length": 1674.7916870117188,
       "epoch": 0.456,
-      "grad_norm": 6.218928657554448,
-      "kl": 1.31640625,
+      "grad_norm": 9.30894413034277,
+      "kl": 0.33447265625,
       "learning_rate": 6.950195628537299e-07,
-      "loss": 0.3387,
-      "reward": 0.859375,
-      "reward_std": 0.22917009145021439,
-      "rewards/tag_count_reward": 0.859375,
+      "loss": 0.0056,
+      "reward": 0.5625,
+      "reward_std": 0.4269454777240753,
+      "rewards/accuracy_reward": 0.5625,
       "step": 456
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1184.5625305175781,
+      "completion_length": 1772.9583740234375,
       "epoch": 0.457,
-      "grad_norm": 10.80209930784645,
-      "kl": 1.1328125,
+      "grad_norm": 8.859278642081595,
+      "kl": 0.2724609375,
       "learning_rate": 6.935316891841315e-07,
-      "loss": 0.3945,
-      "reward": 0.8541666865348816,
-      "reward_std": 0.21569664776325226,
-      "rewards/tag_count_reward": 0.8541666865348816,
+      "loss": 0.0387,
+      "reward": 0.4583333432674408,
+      "reward_std": 0.4314185529947281,
+      "rewards/accuracy_reward": 0.4583333432674408,
       "step": 457
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1348.3125610351562,
+      "completion_length": 1648.7500610351562,
       "epoch": 0.458,
-      "grad_norm": 11.179542694954074,
-      "kl": 1.064453125,
+      "grad_norm": 6.0684680460184035,
+      "kl": 0.28271484375,
       "learning_rate": 6.920420666261961e-07,
-      "loss": 0.4237,
-      "reward": 0.8020833432674408,
-      "reward_std": 0.23151954263448715,
-      "rewards/tag_count_reward": 0.8020833432674408,
+      "loss": 0.0382,
+      "reward": 0.7291666865348816,
+      "reward_std": 0.26679350435733795,
+      "rewards/accuracy_reward": 0.7291666865348816,
       "step": 458
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1252.3958740234375,
+      "completion_length": 1955.8125610351562,
       "epoch": 0.459,
-      "grad_norm": 7.894711801824028,
-      "kl": 1.07421875,
+      "grad_norm": 2.715815651019535,
+      "kl": 0.4521484375,
       "learning_rate": 6.905507133305047e-07,
-      "loss": 0.3084,
-      "reward": 0.875,
-      "reward_std": 0.1687939241528511,
-      "rewards/tag_count_reward": 0.875,
+      "loss": 0.0077,
+      "reward": 0.2916666716337204,
+      "reward_std": 0.3234458789229393,
+      "rewards/accuracy_reward": 0.2916666716337204,
       "step": 459
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1284.666748046875,
+      "completion_length": 1908.5625,
       "epoch": 0.46,
-      "grad_norm": 10.218837999976117,
-      "kl": 1.2734375,
+      "grad_norm": 3.7788449830087685,
+      "kl": 0.3095703125,
       "learning_rate": 6.890576474687263e-07,
-      "loss": 0.3575,
-      "reward": 0.8697916865348816,
-      "reward_std": 0.19752872735261917,
-      "rewards/tag_count_reward": 0.8697916865348816,
+      "loss": 0.0263,
+      "reward": 0.5625,
+      "reward_std": 0.3139677122235298,
+      "rewards/accuracy_reward": 0.5625,
       "step": 460
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1536.2916870117188,
+      "completion_length": 1978.5833740234375,
       "epoch": 0.461,
-      "grad_norm": 6.28405434545915,
-      "kl": 1.55078125,
+      "grad_norm": 4.888131430904844,
+      "kl": 0.37109375,
       "learning_rate": 6.875628872333975e-07,
-      "loss": 0.293,
-      "reward": 0.796875,
-      "reward_std": 0.22817014157772064,
-      "rewards/tag_count_reward": 0.796875,
+      "loss": 0.0417,
+      "reward": 0.3958333432674408,
+      "reward_std": 0.4156641364097595,
+      "rewards/accuracy_reward": 0.3958333432674408,
       "step": 461
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1360.75,
+      "completion_length": 1975.166748046875,
       "epoch": 0.462,
-      "grad_norm": 8.457933807258653,
-      "kl": 1.58203125,
+      "grad_norm": 3.918866503525673,
+      "kl": 0.2841796875,
       "learning_rate": 6.860664508377001e-07,
-      "loss": 0.2901,
-      "reward": 0.8489583432674408,
-      "reward_std": 0.18343394994735718,
-      "rewards/tag_count_reward": 0.8489583432674408,
+      "loss": 0.03,
+      "reward": 0.2916666716337204,
+      "reward_std": 0.45400412380695343,
+      "rewards/accuracy_reward": 0.2916666716337204,
       "step": 462
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1370.229248046875,
+      "completion_length": 1896.9375610351562,
       "epoch": 0.463,
-      "grad_norm": 6.243040180753141,
-      "kl": 1.65625,
+      "grad_norm": 3.745663739533198,
+      "kl": 0.5,
       "learning_rate": 6.84568356515239e-07,
-      "loss": 0.4428,
-      "reward": 0.78125,
-      "reward_std": 0.2227114662528038,
-      "rewards/tag_count_reward": 0.78125,
+      "loss": 0.0304,
+      "reward": 0.3958333432674408,
+      "reward_std": 0.4972747564315796,
+      "rewards/accuracy_reward": 0.3958333432674408,
       "step": 463
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1128.0416870117188,
+      "completion_length": 1888.6666870117188,
       "epoch": 0.464,
-      "grad_norm": 6.440456811434236,
-      "kl": 1.296875,
+      "grad_norm": 5.084426791386579,
+      "kl": 0.330078125,
       "learning_rate": 6.83068622519821e-07,
-      "loss": 0.5212,
-      "reward": 0.8437500298023224,
-      "reward_std": 0.20314887166023254,
-      "rewards/tag_count_reward": 0.8437500298023224,
+      "loss": 0.0219,
+      "reward": 0.5833333432674408,
+      "reward_std": 0.5029153972864151,
+      "rewards/accuracy_reward": 0.5833333432674408,
       "step": 464
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 864.9375,
+      "completion_length": 1826.3541870117188,
       "epoch": 0.465,
-      "grad_norm": 6.887498046550961,
-      "kl": 1.03515625,
+      "grad_norm": 3.4788391447379237,
+      "kl": 0.4267578125,
       "learning_rate": 6.815672671252315e-07,
-      "loss": 0.5915,
-      "reward": 0.8593750298023224,
-      "reward_std": 0.17061126232147217,
-      "rewards/tag_count_reward": 0.8593750298023224,
+      "loss": 0.0132,
+      "reward": 0.5416666865348816,
+      "reward_std": 0.3805558532476425,
+      "rewards/accuracy_reward": 0.5416666865348816,
       "step": 465
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 937.3333435058594,
+      "completion_length": 1787.7916870117188,
       "epoch": 0.466,
-      "grad_norm": 5.250772904790437,
-      "kl": 1.197265625,
+      "grad_norm": 7.7897822174518065,
+      "kl": 0.73828125,
       "learning_rate": 6.800643086250121e-07,
-      "loss": 0.6632,
-      "reward": 0.8385416865348816,
-      "reward_std": 0.1986435353755951,
-      "rewards/tag_count_reward": 0.8385416865348816,
+      "loss": 0.0099,
+      "reward": 0.5833333432674408,
+      "reward_std": 0.43888527154922485,
+      "rewards/accuracy_reward": 0.5833333432674408,
       "step": 466
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 991.8541870117188,
+      "completion_length": 1757.8958740234375,
       "epoch": 0.467,
-      "grad_norm": 11.570194784069281,
-      "kl": 1.306640625,
+      "grad_norm": 7.096603166178015,
+      "kl": 0.400390625,
       "learning_rate": 6.78559765332238e-07,
-      "loss": 0.5321,
-      "reward": 0.8177083730697632,
-      "reward_std": 0.2344372197985649,
-      "rewards/tag_count_reward": 0.8177083730697632,
+      "loss": -0.0173,
+      "reward": 0.4375000149011612,
+      "reward_std": 0.43824973702430725,
+      "rewards/accuracy_reward": 0.4375000149011612,
       "step": 467
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 824.4375,
+      "completion_length": 1735.8333740234375,
       "epoch": 0.468,
-      "grad_norm": 6.672446715960221,
-      "kl": 0.978515625,
+      "grad_norm": 5.676380156255503,
+      "kl": 0.40234375,
       "learning_rate": 6.770536555792944e-07,
-      "loss": 0.6408,
-      "reward": 0.8229166865348816,
-      "reward_std": 0.23008184880018234,
-      "rewards/tag_count_reward": 0.8229166865348816,
+      "loss": -0.0578,
+      "reward": 0.5416666865348816,
+      "reward_std": 0.45455068349838257,
+      "rewards/accuracy_reward": 0.5416666865348816,
       "step": 468
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 804.9583740234375,
+      "completion_length": 1839.3333740234375,
       "epoch": 0.469,
-      "grad_norm": 7.585398445994687,
-      "kl": 1.1796875,
+      "grad_norm": 6.981060564038391,
+      "kl": 0.4052734375,
       "learning_rate": 6.755459977176532e-07,
-      "loss": 0.5452,
-      "reward": 0.8489583432674408,
-      "reward_std": 0.19954737275838852,
-      "rewards/tag_count_reward": 0.8489583432674408,
+      "loss": 0.0071,
+      "reward": 0.625,
+      "reward_std": 0.364890418946743,
+      "rewards/accuracy_reward": 0.625,
       "step": 469
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 623.6458435058594,
+      "completion_length": 1510.5416870117188,
       "epoch": 0.47,
-      "grad_norm": 7.801702501059949,
-      "kl": 0.6171875,
+      "grad_norm": 5.303637983280692,
+      "kl": 0.3798828125,
       "learning_rate": 6.740368101176495e-07,
-      "loss": 0.5407,
-      "reward": 0.8906250298023224,
-      "reward_std": 0.20231406390666962,
-      "rewards/tag_count_reward": 0.8906250298023224,
+      "loss": -0.0377,
+      "reward": 0.6458333432674408,
+      "reward_std": 0.4565621018409729,
+      "rewards/accuracy_reward": 0.6458333432674408,
       "step": 470
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 767.7291870117188,
+      "completion_length": 1610.8333740234375,
       "epoch": 0.471,
-      "grad_norm": 11.038095688888431,
-      "kl": 0.828125,
+      "grad_norm": 6.746723452130682,
+      "kl": 0.640625,
       "learning_rate": 6.725261111682584e-07,
-      "loss": 0.8531,
-      "reward": 0.8958333432674408,
-      "reward_std": 0.18303204327821732,
-      "rewards/tag_count_reward": 0.8958333432674408,
+      "loss": 0.0332,
+      "reward": 0.8541666865348816,
+      "reward_std": 0.3076914772391319,
+      "rewards/accuracy_reward": 0.8541666865348816,
       "step": 471
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 697.7083435058594,
+      "completion_length": 1081.2292175292969,
       "epoch": 0.472,
-      "grad_norm": 12.9836550475911,
-      "kl": 0.697265625,
+      "grad_norm": 8.919286277143206,
+      "kl": 1.30859375,
       "learning_rate": 6.710139192768694e-07,
-      "loss": 0.5237,
-      "reward": 0.8906250298023224,
-      "reward_std": 0.2117478996515274,
-      "rewards/tag_count_reward": 0.8906250298023224,
+      "loss": 0.0607,
+      "reward": 0.3750000111758709,
+      "reward_std": 0.3905205726623535,
+      "rewards/accuracy_reward": 0.3750000111758709,
       "step": 472
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 719.5625305175781,
+      "completion_length": 1561.3750610351562,
       "epoch": 0.473,
-      "grad_norm": 65.68918290296943,
-      "kl": 0.767578125,
+      "grad_norm": 12.73429381997977,
+      "kl": 1.26953125,
       "learning_rate": 6.695002528690639e-07,
-      "loss": 0.7192,
-      "reward": 0.9166666865348816,
-      "reward_std": 0.17281758040189743,
-      "rewards/tag_count_reward": 0.9166666865348816,
+      "loss": -0.0162,
+      "reward": 0.7083333730697632,
+      "reward_std": 0.34349535405635834,
+      "rewards/accuracy_reward": 0.7083333730697632,
       "step": 473
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 815.1458435058594,
+      "completion_length": 1254.3333740234375,
       "epoch": 0.474,
-      "grad_norm": 96.15686083214344,
-      "kl": 2.8984375,
+      "grad_norm": 4.998632166138478,
+      "kl": 0.8984375,
       "learning_rate": 6.679851303883891e-07,
-      "loss": 1.0024,
-      "reward": 0.8802083730697632,
-      "reward_std": 0.21328985691070557,
-      "rewards/tag_count_reward": 0.8802083730697632,
+      "loss": 0.0465,
+      "reward": 0.75,
+      "reward_std": 0.4018019139766693,
+      "rewards/accuracy_reward": 0.75,
       "step": 474
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 737.3541870117188,
+      "completion_length": 1290.8333435058594,
       "epoch": 0.475,
-      "grad_norm": 47.91657008217291,
-      "kl": 2.21875,
+      "grad_norm": 10.134650347710673,
+      "kl": 0.974609375,
       "learning_rate": 6.664685702961344e-07,
-      "loss": 0.7382,
-      "reward": 0.8802083730697632,
-      "reward_std": 0.19346709549427032,
-      "rewards/tag_count_reward": 0.8802083730697632,
+      "loss": 0.2149,
+      "reward": 0.7291666865348816,
+      "reward_std": 0.4112800806760788,
+      "rewards/accuracy_reward": 0.7291666865348816,
       "step": 475
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 692.2916870117188,
+      "completion_length": 1658.8958740234375,
       "epoch": 0.476,
-      "grad_norm": 257.53632873324773,
-      "kl": 4.984375,
+      "grad_norm": 1.7816552128072431,
+      "kl": 0.888671875,
       "learning_rate": 6.649505910711058e-07,
-      "loss": 0.9263,
-      "reward": 0.8697916865348816,
-      "reward_std": 0.21693718433380127,
-      "rewards/tag_count_reward": 0.8697916865348816,
+      "loss": -0.1089,
+      "reward": 0.4791666716337204,
+      "reward_std": 0.5092880129814148,
+      "rewards/accuracy_reward": 0.4791666716337204,
       "step": 476
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 849.0208435058594,
+      "completion_length": 1734.4375610351562,
       "epoch": 0.477,
-      "grad_norm": 97.85672791835103,
-      "kl": 2.6875,
+      "grad_norm": 1.9516105257184626,
+      "kl": 0.98828125,
       "learning_rate": 6.634312112094013e-07,
-      "loss": 0.5899,
-      "reward": 0.8177083432674408,
-      "reward_std": 0.23917385935783386,
-      "rewards/tag_count_reward": 0.8177083432674408,
+      "loss": -0.0349,
+      "reward": 0.1458333395421505,
+      "reward_std": 0.3547166883945465,
+      "rewards/accuracy_reward": 0.1458333395421505,
       "step": 477
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 730.5625,
+      "completion_length": 1788.2708740234375,
       "epoch": 0.478,
-      "grad_norm": 156.19302478120434,
-      "kl": 1.4296875,
+      "grad_norm": 2.742808701536483,
+      "kl": 0.794921875,
       "learning_rate": 6.619104492241847e-07,
-      "loss": 0.6196,
-      "reward": 0.8958333730697632,
-      "reward_std": 0.19614407420158386,
-      "rewards/tag_count_reward": 0.8958333730697632,
+      "loss": 0.0767,
+      "reward": 0.1458333395421505,
+      "reward_std": 0.3076914846897125,
+      "rewards/accuracy_reward": 0.1458333395421505,
       "step": 478
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 574.5416870117188,
+      "completion_length": 1968.1875610351562,
       "epoch": 0.479,
-      "grad_norm": 103.28895867513401,
-      "kl": 1.53125,
+      "grad_norm": 0.5031059494747281,
+      "kl": 0.916015625,
       "learning_rate": 6.603883236454612e-07,
-      "loss": 0.4779,
-      "reward": 0.8645833432674408,
-      "reward_std": 0.18251947313547134,
-      "rewards/tag_count_reward": 0.8645833432674408,
+      "loss": 0.0408,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 479
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 679.7708740234375,
+      "completion_length": 1939.2708740234375,
       "epoch": 0.48,
-      "grad_norm": 124.83742052045773,
-      "kl": 4.7421875,
+      "grad_norm": 4.828722612535397,
+      "kl": 0.96484375,
       "learning_rate": 6.588648530198504e-07,
-      "loss": 0.7442,
-      "reward": 0.8958333432674408,
-      "reward_std": 0.19233327358961105,
-      "rewards/tag_count_reward": 0.8958333432674408,
+      "loss": 0.0597,
+      "reward": 0.18750000558793545,
+      "reward_std": 0.19526028633117676,
+      "rewards/accuracy_reward": 0.18750000558793545,
       "step": 480
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 473.1458435058594,
+      "completion_length": 2037.75,
       "epoch": 0.481,
-      "grad_norm": 35.46585217849751,
-      "kl": 1.1640625,
+      "grad_norm": 1.8063635002728748,
+      "kl": 1.18359375,
       "learning_rate": 6.573400559103613e-07,
-      "loss": 0.2258,
-      "reward": 0.9375000298023224,
-      "reward_std": 0.10045047849416733,
-      "rewards/tag_count_reward": 0.9375000298023224,
+      "loss": 0.0493,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 481
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 436.4583435058594,
+      "completion_length": 2029.2708740234375,
       "epoch": 0.482,
-      "grad_norm": 22.23612829381126,
-      "kl": 0.796875,
+      "grad_norm": 1.0384822326056398,
+      "kl": 1.171875,
       "learning_rate": 6.558139508961654e-07,
-      "loss": 0.3157,
-      "reward": 0.953125,
-      "reward_std": 0.12124212644994259,
-      "rewards/tag_count_reward": 0.953125,
+      "loss": 0.0459,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 482
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 388.62501525878906,
+      "completion_length": 2048.0,
       "epoch": 0.483,
-      "grad_norm": 11.049648561597081,
-      "kl": 0.7451171875,
+      "grad_norm": 0.8511065096535151,
+      "kl": 1.044921875,
       "learning_rate": 6.542865565723707e-07,
-      "loss": 0.1719,
-      "reward": 0.9635416865348816,
-      "reward_std": 0.1110176295042038,
-      "rewards/tag_count_reward": 0.9635416865348816,
+      "loss": 0.0418,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 483
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 444.0208435058594,
+      "completion_length": 2048.0,
       "epoch": 0.484,
-      "grad_norm": 17.789594235685367,
-      "kl": 0.91552734375,
+      "grad_norm": 0.06119305329093999,
+      "kl": 0.904296875,
       "learning_rate": 6.527578915497951e-07,
-      "loss": 0.2053,
-      "reward": 0.9739583730697632,
-      "reward_std": 0.06435108184814453,
-      "rewards/tag_count_reward": 0.9739583730697632,
+      "loss": 0.0362,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 484
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 368.8541717529297,
+      "completion_length": 2048.0,
       "epoch": 0.485,
-      "grad_norm": 2.529096020385133,
-      "kl": 0.20751953125,
+      "grad_norm": 0.053192133878999244,
+      "kl": 0.970703125,
       "learning_rate": 6.512279744547392e-07,
-      "loss": 0.0085,
-      "reward": 1.0,
+      "loss": 0.0388,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 485
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 415.62501525878906,
+      "completion_length": 2048.0,
       "epoch": 0.486,
-      "grad_norm": 19.63053426156352,
-      "kl": 1.0244140625,
+      "grad_norm": 0.05804534758219856,
+      "kl": 0.947265625,
       "learning_rate": 6.496968239287603e-07,
-      "loss": 0.1081,
-      "reward": 0.9322916865348816,
-      "reward_std": 0.14217785745859146,
-      "rewards/tag_count_reward": 0.9322916865348816,
+      "loss": 0.0379,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 486
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 439.8333435058594,
+      "completion_length": 2048.0,
       "epoch": 0.487,
-      "grad_norm": 10.97524555021414,
-      "kl": 1.1123046875,
+      "grad_norm": 0.8975410096403259,
+      "kl": 1.1015625,
       "learning_rate": 6.481644586284442e-07,
-      "loss": 0.0812,
-      "reward": 0.8750000298023224,
-      "reward_std": 0.20938248187303543,
-      "rewards/tag_count_reward": 0.8750000298023224,
+      "loss": 0.0441,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 487
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 399.47918701171875,
+      "completion_length": 2048.0,
       "epoch": 0.488,
-      "grad_norm": 9.925768908740274,
-      "kl": 0.724609375,
+      "grad_norm": 0.08760393506371271,
+      "kl": 0.93359375,
       "learning_rate": 6.466308972251785e-07,
-      "loss": 0.1406,
-      "reward": 0.8541666865348816,
-      "reward_std": 0.21116547286510468,
-      "rewards/tag_count_reward": 0.8541666865348816,
+      "loss": 0.0374,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 488
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 546.2708435058594,
+      "completion_length": 2048.0,
       "epoch": 0.489,
-      "grad_norm": 9.12846183905699,
-      "kl": 0.9140625,
+      "grad_norm": 0.0626953791070745,
+      "kl": 0.89453125,
       "learning_rate": 6.45096158404925e-07,
-      "loss": 0.1185,
-      "reward": 0.75,
-      "reward_std": 0.22956441342830658,
-      "rewards/tag_count_reward": 0.75,
+      "loss": 0.0358,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 489
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 466.31251525878906,
+      "completion_length": 2048.0,
       "epoch": 0.49,
-      "grad_norm": 6.830735300703454,
-      "kl": 0.9296875,
+      "grad_norm": 0.04983456105158953,
+      "kl": 0.9453125,
       "learning_rate": 6.435602608679916e-07,
-      "loss": 0.0974,
-      "reward": 0.7760416865348816,
-      "reward_std": 0.22821441292762756,
-      "rewards/tag_count_reward": 0.7760416865348816,
+      "loss": 0.0378,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 490
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 482.18751525878906,
+      "completion_length": 2048.0,
       "epoch": 0.491,
-      "grad_norm": 15.773366828026825,
-      "kl": 0.88671875,
+      "grad_norm": 0.09240618022822153,
+      "kl": 0.966796875,
       "learning_rate": 6.420232233288055e-07,
-      "loss": 0.1663,
-      "reward": 0.8177083432674408,
-      "reward_std": 0.2563057094812393,
-      "rewards/tag_count_reward": 0.8177083432674408,
+      "loss": 0.0386,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 491
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 426.91668701171875,
+      "completion_length": 2048.0,
       "epoch": 0.492,
-      "grad_norm": 32.803915126715765,
-      "kl": 1.70703125,
+      "grad_norm": 0.10380441805044174,
+      "kl": 0.927734375,
       "learning_rate": 6.404850645156841e-07,
-      "loss": 0.0887,
-      "reward": 0.8489583432674408,
-      "reward_std": 0.20587080717086792,
-      "rewards/tag_count_reward": 0.8489583432674408,
+      "loss": 0.0372,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 492
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 436.7083435058594,
+      "completion_length": 2048.0,
       "epoch": 0.493,
-      "grad_norm": 39.13983549538198,
-      "kl": 2.1015625,
+      "grad_norm": 0.041705880505725615,
+      "kl": 1.0234375,
       "learning_rate": 6.389458031706068e-07,
-      "loss": 0.1392,
-      "reward": 0.8593750298023224,
-      "reward_std": 0.2021520510315895,
-      "rewards/tag_count_reward": 0.8593750298023224,
+      "loss": 0.0408,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 493
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 455.3958435058594,
+      "completion_length": 2048.0,
       "epoch": 0.494,
-      "grad_norm": 12.80499937349423,
-      "kl": 1.44140625,
+      "grad_norm": 0.048843178698266126,
+      "kl": 0.880859375,
       "learning_rate": 6.374054580489873e-07,
-      "loss": 0.1327,
-      "reward": 0.8125000298023224,
-      "reward_std": 0.2328476831316948,
-      "rewards/tag_count_reward": 0.8125000298023224,
+      "loss": 0.0352,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 494
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 491.8125,
+      "completion_length": 2048.0,
       "epoch": 0.495,
-      "grad_norm": 21.581406879814047,
-      "kl": 1.51953125,
+      "grad_norm": 0.11703210127025315,
+      "kl": 0.9453125,
       "learning_rate": 6.358640479194451e-07,
-      "loss": 0.1651,
-      "reward": 0.8333333730697632,
-      "reward_std": 0.22698140144348145,
-      "rewards/tag_count_reward": 0.8333333730697632,
+      "loss": 0.0378,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 495
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 495.04168701171875,
+      "completion_length": 2048.0,
       "epoch": 0.496,
-      "grad_norm": 11.910719077247448,
-      "kl": 1.28515625,
+      "grad_norm": 0.042189399643107806,
+      "kl": 1.001953125,
       "learning_rate": 6.343215915635761e-07,
-      "loss": 0.1314,
-      "reward": 0.7760416865348816,
-      "reward_std": 0.22269393503665924,
-      "rewards/tag_count_reward": 0.7760416865348816,
+      "loss": 0.0401,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 496
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 423.04168701171875,
+      "completion_length": 2048.0,
       "epoch": 0.497,
-      "grad_norm": 18.119709925914574,
-      "kl": 1.55859375,
+      "grad_norm": 0.04627293216327344,
+      "kl": 0.982421875,
       "learning_rate": 6.327781077757241e-07,
-      "loss": 0.1089,
-      "reward": 0.7864583432674408,
-      "reward_std": 0.22677022963762283,
-      "rewards/tag_count_reward": 0.7864583432674408,
+      "loss": 0.0393,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 497
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 411.8125,
+      "completion_length": 2048.0,
       "epoch": 0.498,
-      "grad_norm": 8.939652278302464,
-      "kl": 0.693359375,
+      "grad_norm": 0.24244219069003248,
+      "kl": 0.97265625,
       "learning_rate": 6.31233615362752e-07,
-      "loss": 0.06,
-      "reward": 0.90625,
-      "reward_std": 0.1671900376677513,
-      "rewards/tag_count_reward": 0.90625,
+      "loss": 0.0389,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 498
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 475.50001525878906,
+      "completion_length": 2048.0,
       "epoch": 0.499,
-      "grad_norm": 242.16416813046348,
-      "kl": 3.513671875,
+      "grad_norm": 0.03232872190877127,
+      "kl": 0.908203125,
       "learning_rate": 6.296881331438126e-07,
-      "loss": 0.3017,
-      "reward": 0.8541666865348816,
-      "reward_std": 0.21013373881578445,
-      "rewards/tag_count_reward": 0.8541666865348816,
+      "loss": 0.0364,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 499
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 478.75001525878906,
+      "completion_length": 2048.0,
       "epoch": 0.5,
-      "grad_norm": 9.289622996609156,
-      "kl": 1.484375,
+      "grad_norm": 0.16529553508301678,
+      "kl": 0.9609375,
       "learning_rate": 6.281416799501187e-07,
-      "loss": 0.1325,
-      "reward": 0.8333333432674408,
-      "reward_std": 0.2499879226088524,
-      "rewards/tag_count_reward": 0.8333333432674408,
+      "loss": 0.0384,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 500
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 434.0833435058594,
+      "completion_length": 2048.0,
       "epoch": 0.501,
-      "grad_norm": 72.80609003854057,
-      "kl": 2.296875,
+      "grad_norm": 0.1032381569198689,
+      "kl": 0.908203125,
       "learning_rate": 6.265942746247146e-07,
-      "loss": 0.3123,
-      "reward": 0.90625,
-      "reward_std": 0.1826242059469223,
-      "rewards/tag_count_reward": 0.90625,
+      "loss": 0.0363,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 501
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 397.0208435058594,
+      "completion_length": 2048.0,
       "epoch": 0.502,
-      "grad_norm": 10.01226903410698,
-      "kl": 1.0859375,
+      "grad_norm": 0.07041796875545865,
+      "kl": 0.923828125,
       "learning_rate": 6.25045936022246e-07,
-      "loss": 0.1132,
-      "reward": 0.9114583432674408,
-      "reward_std": 0.18020494282245636,
-      "rewards/tag_count_reward": 0.9114583432674408,
+      "loss": 0.037,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 502
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 378.6041717529297,
+      "completion_length": 2048.0,
       "epoch": 0.503,
-      "grad_norm": 25.30471139536292,
-      "kl": 1.4609375,
+      "grad_norm": 0.07726463588526927,
+      "kl": 1.0,
       "learning_rate": 6.2349668300873e-07,
-      "loss": 0.1484,
-      "reward": 0.9375000298023224,
-      "reward_std": 0.1506819725036621,
-      "rewards/tag_count_reward": 0.9375000298023224,
+      "loss": 0.04,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 503
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 413.00001525878906,
+      "completion_length": 2048.0,
       "epoch": 0.504,
-      "grad_norm": 26.653952653018365,
-      "kl": 2.2890625,
+      "grad_norm": 0.0636071106409591,
+      "kl": 0.955078125,
       "learning_rate": 6.219465344613258e-07,
-      "loss": 0.2085,
-      "reward": 0.90625,
-      "reward_std": 0.17194338142871857,
-      "rewards/tag_count_reward": 0.90625,
+      "loss": 0.0382,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 504
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 417.0208435058594,
+      "completion_length": 2048.0,
       "epoch": 0.505,
-      "grad_norm": 18.34064468054739,
-      "kl": 1.0390625,
+      "grad_norm": 0.052282449626567314,
+      "kl": 0.908203125,
       "learning_rate": 6.203955092681039e-07,
-      "loss": 0.1238,
-      "reward": 0.9375000298023224,
-      "reward_std": 0.13520457595586777,
-      "rewards/tag_count_reward": 0.9375000298023224,
+      "loss": 0.0363,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 505
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 394.56251525878906,
+      "completion_length": 2048.0,
       "epoch": 0.506,
-      "grad_norm": 14.306057688537363,
-      "kl": 0.4296875,
+      "grad_norm": 0.10307402952589698,
+      "kl": 0.96875,
       "learning_rate": 6.188436263278172e-07,
-      "loss": 0.1227,
-      "reward": 0.9531250298023224,
-      "reward_std": 0.12172887474298477,
-      "rewards/tag_count_reward": 0.9531250298023224,
+      "loss": 0.0388,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 506
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 402.9166717529297,
+      "completion_length": 2048.0,
       "epoch": 0.507,
-      "grad_norm": 43.426631300255,
-      "kl": 1.783203125,
+      "grad_norm": 0.04987121773529549,
+      "kl": 1.01953125,
       "learning_rate": 6.172909045496694e-07,
-      "loss": 0.1454,
-      "reward": 0.9479166865348816,
-      "reward_std": 0.13458874076604843,
-      "rewards/tag_count_reward": 0.9479166865348816,
+      "loss": 0.0407,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 507
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 410.9791717529297,
+      "completion_length": 2048.0,
       "epoch": 0.508,
-      "grad_norm": 8.009705320931293,
-      "kl": 0.3251953125,
+      "grad_norm": 0.11998314919204613,
+      "kl": 0.91796875,
       "learning_rate": 6.157373628530852e-07,
-      "loss": 0.0487,
-      "reward": 0.9531250298023224,
-      "reward_std": 0.14710202813148499,
-      "rewards/tag_count_reward": 0.9531250298023224,
+      "loss": 0.0368,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 508
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 441.77085876464844,
+      "completion_length": 2048.0,
       "epoch": 0.509,
-      "grad_norm": 7.064052309828297,
-      "kl": 0.5234375,
+      "grad_norm": 0.07101611499110734,
+      "kl": 0.876953125,
       "learning_rate": 6.141830201674802e-07,
-      "loss": 0.0775,
-      "reward": 0.96875,
-      "reward_std": 0.07769769430160522,
-      "rewards/tag_count_reward": 0.96875,
+      "loss": 0.0351,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 509
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 459.6666717529297,
+      "completion_length": 2048.0,
       "epoch": 0.51,
-      "grad_norm": 30.244709764967375,
-      "kl": 1.96875,
+      "grad_norm": 0.04434886597577572,
+      "kl": 0.955078125,
       "learning_rate": 6.126278954320294e-07,
-      "loss": 0.2789,
-      "reward": 0.9270833432674408,
-      "reward_std": 0.18332497030496597,
-      "rewards/tag_count_reward": 0.9270833432674408,
+      "loss": 0.0383,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 510
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 415.2083435058594,
+      "completion_length": 2048.0,
       "epoch": 0.511,
-      "grad_norm": 115.24412450112247,
-      "kl": 3.8671875,
+      "grad_norm": 0.0504544903079507,
+      "kl": 0.94140625,
       "learning_rate": 6.11072007595437e-07,
-      "loss": 0.2433,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.07216878235340118,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0376,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 511
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 382.3958435058594,
+      "completion_length": 2048.0,
       "epoch": 0.512,
-      "grad_norm": 195.61340613748922,
-      "kl": 8.140625,
+      "grad_norm": 0.052632281957982245,
+      "kl": 0.955078125,
       "learning_rate": 6.095153756157051e-07,
-      "loss": 0.5186,
-      "reward": 0.9531250298023224,
-      "reward_std": 0.11535455286502838,
-      "rewards/tag_count_reward": 0.9531250298023224,
+      "loss": 0.0382,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 512
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 371.5208435058594,
+      "completion_length": 2048.0,
       "epoch": 0.513,
-      "grad_norm": 16.785721604403555,
-      "kl": 1.45703125,
+      "grad_norm": 0.9866431623247501,
+      "kl": 0.986328125,
       "learning_rate": 6.079580184599032e-07,
-      "loss": 0.084,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.09021097421646118,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.0394,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 513
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 353.4583435058594,
+      "completion_length": 2048.0,
       "epoch": 0.514,
-      "grad_norm": 4.699772991934067,
-      "kl": 0.38330078125,
+      "grad_norm": 0.0824429680262861,
+      "kl": 0.94921875,
       "learning_rate": 6.06399955103937e-07,
-      "loss": 0.0608,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.038,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 514
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 390.2291717529297,
+      "completion_length": 2048.0,
       "epoch": 0.515,
-      "grad_norm": 14.015572428904417,
-      "kl": 0.80859375,
+      "grad_norm": 0.12739513106796405,
+      "kl": 1.03125,
       "learning_rate": 6.048412045323164e-07,
-      "loss": 0.1295,
-      "reward": 0.9531250298023224,
-      "reward_std": 0.12172887474298477,
-      "rewards/tag_count_reward": 0.9531250298023224,
+      "loss": 0.0412,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 515
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 371.0208435058594,
+      "completion_length": 2048.0,
       "epoch": 0.516,
-      "grad_norm": 7.072123039382963,
-      "kl": 0.151611328125,
+      "grad_norm": 0.1489077342598473,
+      "kl": 1.02734375,
       "learning_rate": 6.032817857379256e-07,
-      "loss": 0.0768,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0412,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 516
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 348.2291717529297,
+      "completion_length": 2048.0,
       "epoch": 0.517,
-      "grad_norm": 1.6403662268885855,
-      "kl": 0.15283203125,
+      "grad_norm": 0.03900533985947159,
+      "kl": 0.99609375,
       "learning_rate": 6.017217177217899e-07,
-      "loss": 0.0061,
-      "reward": 1.0,
+      "loss": 0.0399,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 517
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 354.5833435058594,
+      "completion_length": 2048.0,
       "epoch": 0.518,
-      "grad_norm": 494.9613593566199,
-      "kl": 7.6015625,
+      "grad_norm": 0.06199764608077145,
+      "kl": 0.923828125,
       "learning_rate": 6.001610194928464e-07,
-      "loss": 0.6701,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.09021097794175148,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.037,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 518
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 435.29168701171875,
+      "completion_length": 2048.0,
       "epoch": 0.519,
-      "grad_norm": 14.551518722427685,
-      "kl": 0.8330078125,
+      "grad_norm": 0.2545642621153581,
+      "kl": 0.95703125,
       "learning_rate": 5.985997100677103e-07,
-      "loss": 0.0463,
-      "reward": 0.984375,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0382,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 519
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 378.1458435058594,
+      "completion_length": 2048.0,
       "epoch": 0.52,
-      "grad_norm": 12.964649241316112,
-      "kl": 0.915283203125,
+      "grad_norm": 0.05280570291811574,
+      "kl": 0.951171875,
       "learning_rate": 5.97037808470444e-07,
-      "loss": 0.0657,
-      "reward": 0.984375,
-      "reward_std": 0.03884884715080261,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.038,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 520
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 385.9583435058594,
+      "completion_length": 2048.0,
       "epoch": 0.521,
-      "grad_norm": 51.96366845104974,
-      "kl": 1.982421875,
+      "grad_norm": 0.04147416353313374,
+      "kl": 0.95703125,
       "learning_rate": 5.954753337323259e-07,
-      "loss": 0.1629,
-      "reward": 0.9687500298023224,
-      "reward_std": 0.08474057167768478,
-      "rewards/tag_count_reward": 0.9687500298023224,
+      "loss": 0.0382,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 521
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 379.1666717529297,
+      "completion_length": 2048.0,
       "epoch": 0.522,
-      "grad_norm": 10.509987238838747,
-      "kl": 0.7578125,
+      "grad_norm": 0.23858861469758186,
+      "kl": 0.92578125,
       "learning_rate": 5.939123048916173e-07,
-      "loss": 0.0965,
-      "reward": 0.9687500298023224,
-      "reward_std": 0.10825317353010178,
-      "rewards/tag_count_reward": 0.9687500298023224,
+      "loss": 0.0371,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 522
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 366.7916717529297,
+      "completion_length": 2048.0,
       "epoch": 0.523,
-      "grad_norm": 8.71809976619088,
-      "kl": 0.42626953125,
+      "grad_norm": 0.1028462165894241,
+      "kl": 0.986328125,
       "learning_rate": 5.923487409933315e-07,
-      "loss": 0.0853,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.09021097421646118,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.0394,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 523
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 409.75001525878906,
+      "completion_length": 2048.0,
       "epoch": 0.524,
-      "grad_norm": 26.138982212177606,
-      "kl": 1.822265625,
+      "grad_norm": 0.059631743074659675,
+      "kl": 0.9765625,
       "learning_rate": 5.907846610890011e-07,
-      "loss": 0.1788,
-      "reward": 0.9583333432674408,
-      "reward_std": 0.11287746578454971,
-      "rewards/tag_count_reward": 0.9583333432674408,
+      "loss": 0.0391,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 524
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 403.0416717529297,
+      "completion_length": 2048.0,
       "epoch": 0.525,
-      "grad_norm": 7.60573189749456,
-      "kl": 0.4453125,
+      "grad_norm": 0.04105016738667662,
+      "kl": 0.923828125,
       "learning_rate": 5.892200842364462e-07,
-      "loss": 0.0792,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.06317693740129471,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.0369,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 525
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 391.6666717529297,
+      "completion_length": 2048.0,
       "epoch": 0.526,
-      "grad_norm": 162.43944905543728,
-      "kl": 5.5390625,
+      "grad_norm": 0.19717446828877913,
+      "kl": 1.015625,
       "learning_rate": 5.87655029499542e-07,
-      "loss": 0.3085,
-      "reward": 0.9427083432674408,
-      "reward_std": 0.09432543441653252,
-      "rewards/tag_count_reward": 0.9427083432674408,
+      "loss": 0.0405,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 526
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 441.8333435058594,
+      "completion_length": 2048.0,
       "epoch": 0.527,
-      "grad_norm": 222.20817265394626,
-      "kl": 6.86328125,
+      "grad_norm": 0.22904683343179097,
+      "kl": 0.88671875,
       "learning_rate": 5.860895159479864e-07,
-      "loss": 0.3774,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.07216878235340118,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0354,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 527
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 467.8958435058594,
+      "completion_length": 2048.0,
       "epoch": 0.528,
-      "grad_norm": 107.24042987401106,
-      "kl": 5.59375,
+      "grad_norm": 0.729340423520361,
+      "kl": 0.96875,
       "learning_rate": 5.845235626570683e-07,
-      "loss": 0.3771,
-      "reward": 0.9427083432674408,
-      "reward_std": 0.14112106710672379,
-      "rewards/tag_count_reward": 0.9427083432674408,
+      "loss": 0.0388,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 528
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 457.8333435058594,
+      "completion_length": 2048.0,
       "epoch": 0.529,
-      "grad_norm": 31.326251648615383,
-      "kl": 1.126953125,
+      "grad_norm": 0.07239429813639346,
+      "kl": 0.943359375,
       "learning_rate": 5.829571887074343e-07,
-      "loss": 0.2654,
-      "reward": 0.9687500298023224,
-      "reward_std": 0.08474057167768478,
-      "rewards/tag_count_reward": 0.9687500298023224,
+      "loss": 0.0378,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 529
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 431.06251525878906,
+      "completion_length": 2048.0,
       "epoch": 0.53,
-      "grad_norm": 8.097526346389188,
-      "kl": 0.5712890625,
+      "grad_norm": 0.04502199450052443,
+      "kl": 0.982421875,
       "learning_rate": 5.813904131848564e-07,
-      "loss": 0.0689,
-      "reward": 0.9635416865348816,
-      "reward_std": 0.12629536539316177,
-      "rewards/tag_count_reward": 0.9635416865348816,
+      "loss": 0.0393,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 530
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 451.0208435058594,
+      "completion_length": 2048.0,
       "epoch": 0.531,
-      "grad_norm": 18.943146834182464,
-      "kl": 0.8515625,
+      "grad_norm": 0.05151775433443019,
+      "kl": 0.923828125,
       "learning_rate": 5.798232551800002e-07,
-      "loss": 0.3034,
-      "reward": 0.9166666865348816,
-      "reward_std": 0.18585222214460373,
-      "rewards/tag_count_reward": 0.9166666865348816,
+      "loss": 0.0369,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 531
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 364.68751525878906,
+      "completion_length": 2048.0,
       "epoch": 0.532,
-      "grad_norm": 8.498932430014783,
-      "kl": 0.5078125,
+      "grad_norm": 0.05196565553132223,
+      "kl": 0.974609375,
       "learning_rate": 5.78255733788191e-07,
-      "loss": 0.0566,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.0749332383275032,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.039,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 532
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 358.81251525878906,
+      "completion_length": 2048.0,
       "epoch": 0.533,
-      "grad_norm": 9.600836590427424,
-      "kl": 0.34765625,
+      "grad_norm": 0.04616552640518809,
+      "kl": 0.96484375,
       "learning_rate": 5.766878681091828e-07,
-      "loss": 0.1632,
-      "reward": 0.9687500298023224,
-      "reward_std": 0.10825317353010178,
-      "rewards/tag_count_reward": 0.9687500298023224,
+      "loss": 0.0386,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 533
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 396.7708435058594,
+      "completion_length": 2048.0,
       "epoch": 0.534,
-      "grad_norm": 27.376458471111953,
-      "kl": 0.9453125,
+      "grad_norm": 0.12463245173178007,
+      "kl": 1.01171875,
       "learning_rate": 5.751196772469237e-07,
-      "loss": 0.1872,
-      "reward": 0.9687500298023224,
-      "reward_std": 0.10825317353010178,
-      "rewards/tag_count_reward": 0.9687500298023224,
+      "loss": 0.0405,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 534
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 391.5208435058594,
+      "completion_length": 2048.0,
       "epoch": 0.535,
-      "grad_norm": 25.511749648224892,
-      "kl": 1.671875,
+      "grad_norm": 0.11835353252503554,
+      "kl": 0.955078125,
       "learning_rate": 5.735511803093248e-07,
-      "loss": 0.1309,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": 0.0382,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 535
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 362.8541717529297,
+      "completion_length": 2048.0,
       "epoch": 0.536,
-      "grad_norm": 98.56527960040326,
-      "kl": 4.28125,
+      "grad_norm": 0.15873855697808406,
+      "kl": 0.962890625,
       "learning_rate": 5.71982396408026e-07,
-      "loss": 0.2705,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0385,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 536
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 395.72918701171875,
+      "completion_length": 2048.0,
       "epoch": 0.537,
-      "grad_norm": 30.228185583257314,
-      "kl": 1.845703125,
+      "grad_norm": 0.10678776322874263,
+      "kl": 0.919921875,
       "learning_rate": 5.704133446581642e-07,
-      "loss": 0.1388,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.04956009238958359,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.0369,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 537
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 424.54168701171875,
+      "completion_length": 2048.0,
       "epoch": 0.538,
-      "grad_norm": 55.78191720577316,
-      "kl": 2.5703125,
+      "grad_norm": 0.15941338493391607,
+      "kl": 0.91015625,
       "learning_rate": 5.688440441781398e-07,
-      "loss": 0.1954,
-      "reward": 0.9687500298023224,
-      "reward_std": 0.0929754376411438,
-      "rewards/tag_count_reward": 0.9687500298023224,
+      "loss": 0.0364,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 538
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 392.50001525878906,
+      "completion_length": 2048.0,
       "epoch": 0.539,
-      "grad_norm": 12.773537553545584,
-      "kl": 0.94921875,
+      "grad_norm": 0.1518779970743927,
+      "kl": 0.890625,
       "learning_rate": 5.672745140893839e-07,
-      "loss": 0.1353,
-      "reward": 0.9687500298023224,
-      "reward_std": 0.08474056795239449,
-      "rewards/tag_count_reward": 0.9687500298023224,
+      "loss": 0.0357,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 539
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 420.7291717529297,
+      "completion_length": 2048.0,
       "epoch": 0.54,
-      "grad_norm": 4.149720436875501,
-      "kl": 0.828125,
+      "grad_norm": 0.15402962511419677,
+      "kl": 0.919921875,
       "learning_rate": 5.657047735161255e-07,
-      "loss": 0.067,
-      "reward": 0.9531250298023224,
-      "reward_std": 0.11773939803242683,
-      "rewards/tag_count_reward": 0.9531250298023224,
+      "loss": 0.0368,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 540
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 377.4791717529297,
+      "completion_length": 2048.0,
       "epoch": 0.541,
-      "grad_norm": 5.7930433922410325,
-      "kl": 0.5126953125,
+      "grad_norm": 0.08249251487309006,
+      "kl": 0.9453125,
       "learning_rate": 5.641348415851577e-07,
-      "loss": 0.052,
-      "reward": 0.9687500298023224,
-      "reward_std": 0.07298427075147629,
-      "rewards/tag_count_reward": 0.9687500298023224,
+      "loss": 0.0378,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 541
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 479.1041717529297,
+      "completion_length": 2048.0,
       "epoch": 0.542,
-      "grad_norm": 16.982699499324443,
-      "kl": 0.515625,
+      "grad_norm": 0.18306485394820884,
+      "kl": 0.84765625,
       "learning_rate": 5.625647374256061e-07,
-      "loss": 0.1876,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.0749332457780838,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.0339,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 542
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 356.4375,
+      "completion_length": 2048.0,
       "epoch": 0.543,
-      "grad_norm": 1.042112146020366,
-      "kl": 0.132568359375,
+      "grad_norm": 0.07219317268615026,
+      "kl": 0.85546875,
       "learning_rate": 5.60994480168694e-07,
-      "loss": 0.006,
-      "reward": 1.0,
+      "loss": 0.0343,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 543
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 420.87501525878906,
+      "completion_length": 2048.0,
       "epoch": 0.544,
-      "grad_norm": 12.484906716391471,
-      "kl": 0.611328125,
+      "grad_norm": 0.19866238949608747,
+      "kl": 0.853515625,
       "learning_rate": 5.594240889475106e-07,
-      "loss": 0.1302,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.05689104273915291,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0341,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 544
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 419.4583435058594,
+      "completion_length": 2048.0,
       "epoch": 0.545,
-      "grad_norm": 8.979713953123971,
-      "kl": 0.5458984375,
+      "grad_norm": 0.27369994691853733,
+      "kl": 0.8203125,
       "learning_rate": 5.578535828967777e-07,
-      "loss": 0.0264,
-      "reward": 1.0,
+      "loss": 0.0328,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 545
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 472.7291717529297,
+      "completion_length": 2048.0,
       "epoch": 0.546,
-      "grad_norm": 11.351801658638019,
-      "kl": 1.1796875,
+      "grad_norm": 0.16025260037652156,
+      "kl": 0.830078125,
       "learning_rate": 5.562829811526154e-07,
-      "loss": 0.2698,
-      "reward": 0.9479166865348816,
-      "reward_std": 0.13466878980398178,
-      "rewards/tag_count_reward": 0.9479166865348816,
+      "loss": 0.0332,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 546
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 414.3125,
+      "completion_length": 2048.0,
       "epoch": 0.547,
-      "grad_norm": 5.840490960004321,
-      "kl": 0.517578125,
+      "grad_norm": 0.25649503964619125,
+      "kl": 0.787109375,
       "learning_rate": 5.547123028523106e-07,
-      "loss": 0.1347,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0315,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 547
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 421.12501525878906,
+      "completion_length": 2048.0,
       "epoch": 0.548,
-      "grad_norm": 31.369294570677745,
-      "kl": 1.8203125,
+      "grad_norm": 0.3323998870780412,
+      "kl": 0.736328125,
       "learning_rate": 5.531415671340826e-07,
-      "loss": 0.1599,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0295,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 548
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 367.3333435058594,
+      "completion_length": 2048.0,
       "epoch": 0.549,
-      "grad_norm": 3.215178196877606,
-      "kl": 0.24951171875,
+      "grad_norm": 0.2924266436502754,
+      "kl": 0.904296875,
       "learning_rate": 5.515707931368507e-07,
-      "loss": 0.011,
-      "reward": 1.0,
+      "loss": 0.0362,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 549
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 407.7083435058594,
+      "completion_length": 2048.0,
       "epoch": 0.55,
-      "grad_norm": 4.109487023976539,
-      "kl": 0.309814453125,
+      "grad_norm": 0.22858745064563565,
+      "kl": 0.828125,
       "learning_rate": 5.5e-07,
-      "loss": 0.0139,
-      "reward": 1.0,
+      "loss": 0.0331,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 550
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 434.8541717529297,
+      "completion_length": 2013.9583740234375,
       "epoch": 0.551,
-      "grad_norm": 5.49398024609441,
-      "kl": 0.248291015625,
+      "grad_norm": 2.5951685091685675,
+      "kl": 0.5537109375,
       "learning_rate": 5.484292068631494e-07,
-      "loss": 0.0506,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0611,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.09731236100196838,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 551
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 411.3333435058594,
+      "completion_length": 2042.3541870117188,
       "epoch": 0.552,
-      "grad_norm": 10.446442147389755,
-      "kl": 0.671875,
+      "grad_norm": 6.440670756118401,
+      "kl": 0.39453125,
       "learning_rate": 5.468584328659172e-07,
-      "loss": 0.0515,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.04865618050098419,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0243,
+      "reward": 0.06250000186264515,
+      "reward_std": 0.21650634706020355,
+      "rewards/accuracy_reward": 0.06250000186264515,
       "step": 552
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 407.9583435058594,
+      "completion_length": 2048.0,
       "epoch": 0.553,
-      "grad_norm": 11.659490993973444,
-      "kl": 0.26318359375,
+      "grad_norm": 0.5358747814012813,
+      "kl": 0.392578125,
       "learning_rate": 5.452876971476896e-07,
-      "loss": 0.0936,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.09021097421646118,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.0157,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 553
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 392.25,
+      "completion_length": 2048.0,
       "epoch": 0.554,
-      "grad_norm": 6.929076125153229,
-      "kl": 0.462890625,
+      "grad_norm": 0.660499023647641,
+      "kl": 0.38671875,
       "learning_rate": 5.437170188473847e-07,
-      "loss": 0.0281,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.04865618050098419,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0155,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 554
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 400.8958435058594,
+      "completion_length": 2047.5833740234375,
       "epoch": 0.555,
-      "grad_norm": 0.2713283619354236,
-      "kl": 0.105712890625,
+      "grad_norm": 2.7116436925104965,
+      "kl": 0.23046875,
       "learning_rate": 5.421464171032224e-07,
-      "loss": 0.0044,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0098,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 555
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 384.9791717529297,
+      "completion_length": 2045.0625,
       "epoch": 0.556,
-      "grad_norm": 10.336134198326514,
-      "kl": 0.837890625,
+      "grad_norm": 0.7635603057315651,
+      "kl": 0.2412109375,
       "learning_rate": 5.405759110524894e-07,
-      "loss": 0.1197,
-      "reward": 0.9739583730697632,
-      "reward_std": 0.06669837608933449,
-      "rewards/tag_count_reward": 0.9739583730697632,
+      "loss": 0.0097,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 556
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 459.8958435058594,
+      "completion_length": 2048.0,
       "epoch": 0.557,
-      "grad_norm": 106.61184551140086,
-      "kl": 3.3203125,
+      "grad_norm": 0.9107168797499324,
+      "kl": 0.20556640625,
       "learning_rate": 5.390055198313061e-07,
-      "loss": 0.256,
-      "reward": 0.984375,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0082,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 557
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 350.85418701171875,
+      "completion_length": 2025.0833740234375,
       "epoch": 0.558,
-      "grad_norm": 29.507973013028842,
-      "kl": 1.607421875,
+      "grad_norm": 3.747437440554043,
+      "kl": 0.20361328125,
       "learning_rate": 5.37435262574394e-07,
-      "loss": 0.1244,
-      "reward": 0.984375,
-      "reward_std": 0.03884884715080261,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0371,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 558
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 422.00001525878906,
+      "completion_length": 2001.6666870117188,
       "epoch": 0.559,
-      "grad_norm": 13.652699765057086,
-      "kl": 1.53125,
+      "grad_norm": 5.3922814637725285,
+      "kl": 0.2041015625,
       "learning_rate": 5.358651584148423e-07,
-      "loss": 0.171,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": 0.0731,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.14433756470680237,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 559
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 416.2291717529297,
+      "completion_length": 1999.9166870117188,
       "epoch": 0.56,
-      "grad_norm": 17.52960900631826,
-      "kl": 0.5546875,
+      "grad_norm": 4.765637029401671,
+      "kl": 0.22705078125,
       "learning_rate": 5.342952264838747e-07,
-      "loss": 0.025,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0084,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.09731236100196838,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 560
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 421.68751525878906,
+      "completion_length": 1936.1458740234375,
       "epoch": 0.561,
-      "grad_norm": 13.64501272217535,
-      "kl": 0.611328125,
+      "grad_norm": 5.210651772539476,
+      "kl": 0.23876953125,
       "learning_rate": 5.32725485910616e-07,
-      "loss": 0.1599,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.07216878235340118,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0391,
+      "reward": 0.08333333395421505,
+      "reward_std": 0.24164992570877075,
+      "rewards/accuracy_reward": 0.08333333395421505,
       "step": 561
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 391.875,
+      "completion_length": 2037.7291870117188,
       "epoch": 0.562,
-      "grad_norm": 2.5491679809231127,
-      "kl": 0.137451171875,
+      "grad_norm": 1.5663926455342851,
+      "kl": 0.18017578125,
       "learning_rate": 5.311559558218603e-07,
-      "loss": 0.0147,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0072,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 562
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 401.31251525878906,
+      "completion_length": 1969.1666870117188,
       "epoch": 0.563,
-      "grad_norm": 4.48586331295426,
-      "kl": 0.208251953125,
+      "grad_norm": 3.610550483813094,
+      "kl": 0.21044921875,
       "learning_rate": 5.295866553418358e-07,
-      "loss": 0.0002,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.054,
+      "reward": 0.10416666977107525,
+      "reward_std": 0.19526028633117676,
+      "rewards/accuracy_reward": 0.10416666977107525,
       "step": 563
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 372.16668701171875,
+      "completion_length": 1908.25,
       "epoch": 0.564,
-      "grad_norm": 4.886429592648346,
-      "kl": 0.556640625,
+      "grad_norm": 6.976235159915834,
+      "kl": 0.24658203125,
       "learning_rate": 5.28017603591974e-07,
-      "loss": 0.0217,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0818,
+      "reward": 0.2083333432674408,
+      "reward_std": 0.3391112759709358,
+      "rewards/accuracy_reward": 0.2083333432674408,
       "step": 564
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 405.7083435058594,
+      "completion_length": 1856.6875610351562,
       "epoch": 0.565,
-      "grad_norm": 14.789814425659378,
-      "kl": 0.94189453125,
+      "grad_norm": 5.172758412832295,
+      "kl": 0.345703125,
       "learning_rate": 5.264488196906752e-07,
-      "loss": 0.0884,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0558,
+      "reward": 0.2708333432674408,
+      "reward_std": 0.4036279767751694,
+      "rewards/accuracy_reward": 0.2708333432674408,
       "step": 565
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 398.25001525878906,
+      "completion_length": 1732.0,
       "epoch": 0.566,
-      "grad_norm": 32.17134982668231,
-      "kl": 2.3115234375,
+      "grad_norm": 11.13458112007908,
+      "kl": 0.3857421875,
       "learning_rate": 5.248803227530763e-07,
-      "loss": 0.1142,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.04956009238958359,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.155,
+      "reward": 0.3333333432674408,
+      "reward_std": 0.364890418946743,
+      "rewards/accuracy_reward": 0.3333333432674408,
       "step": 566
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 383.47918701171875,
+      "completion_length": 1846.7291870117188,
       "epoch": 0.567,
-      "grad_norm": 12.737701967241774,
-      "kl": 0.9169921875,
+      "grad_norm": 3.050033056914542,
+      "kl": 0.587890625,
       "learning_rate": 5.233121318908173e-07,
-      "loss": 0.1053,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0885,
+      "reward": 0.0833333358168602,
+      "reward_std": 0.19462472200393677,
+      "rewards/accuracy_reward": 0.0833333358168602,
       "step": 567
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 358.43751525878906,
+      "completion_length": 1708.3125,
       "epoch": 0.568,
-      "grad_norm": 21.185906835712984,
-      "kl": 1.016845703125,
+      "grad_norm": 10.23545731374021,
+      "kl": 0.884765625,
       "learning_rate": 5.21744266211809e-07,
-      "loss": 0.2176,
-      "reward": 0.984375,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0575,
+      "reward": 0.2083333432674408,
+      "reward_std": 0.4056393951177597,
+      "rewards/accuracy_reward": 0.2083333432674408,
       "step": 568
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 415.3333435058594,
+      "completion_length": 1558.8333740234375,
       "epoch": 0.569,
-      "grad_norm": 17.39490993968257,
-      "kl": 0.84033203125,
+      "grad_norm": 9.041136539329777,
+      "kl": 0.75390625,
       "learning_rate": 5.2017674482e-07,
-      "loss": 0.0687,
-      "reward": 0.984375,
-      "reward_std": 0.03884884715080261,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0243,
+      "reward": 0.33333333395421505,
+      "reward_std": 0.18523553758859634,
+      "rewards/accuracy_reward": 0.33333333395421505,
       "step": 569
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 397.0208435058594,
+      "completion_length": 1604.8333740234375,
       "epoch": 0.57,
-      "grad_norm": 6.288612720147936,
-      "kl": 0.385498046875,
+      "grad_norm": 8.404915076944258,
+      "kl": 0.80859375,
       "learning_rate": 5.186095868151436e-07,
-      "loss": 0.0489,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.04865618050098419,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0466,
+      "reward": 0.229166679084301,
+      "reward_std": 0.42066924273967743,
+      "rewards/accuracy_reward": 0.229166679084301,
       "step": 570
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 397.31251525878906,
+      "completion_length": 1623.4166870117188,
       "epoch": 0.571,
-      "grad_norm": 8.595110401084613,
-      "kl": 0.478515625,
+      "grad_norm": 6.263039160522345,
+      "kl": 0.75,
       "learning_rate": 5.170428112925659e-07,
-      "loss": 0.0841,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.0749332383275032,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.1115,
+      "reward": 0.4583333432674408,
+      "reward_std": 0.5036473274230957,
+      "rewards/accuracy_reward": 0.4583333432674408,
       "step": 571
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 372.62501525878906,
+      "completion_length": 1580.8750610351562,
       "epoch": 0.572,
-      "grad_norm": 0.5249421203742181,
-      "kl": 0.10546875,
+      "grad_norm": 9.591171880148007,
+      "kl": 0.66796875,
       "learning_rate": 5.154764373429315e-07,
-      "loss": 0.0044,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0502,
+      "reward": 0.375,
+      "reward_std": 0.4778681844472885,
+      "rewards/accuracy_reward": 0.375,
       "step": 572
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 359.9583435058594,
+      "completion_length": 1650.1458740234375,
       "epoch": 0.573,
-      "grad_norm": 18.64973350155189,
-      "kl": 0.6201171875,
+      "grad_norm": 6.057225912980767,
+      "kl": 0.5185546875,
       "learning_rate": 5.139104840520135e-07,
-      "loss": 0.0493,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.113,
+      "reward": 0.2708333358168602,
+      "reward_std": 0.4370592087507248,
+      "rewards/accuracy_reward": 0.2708333358168602,
       "step": 573
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 372.29168701171875,
+      "completion_length": 1710.0416870117188,
       "epoch": 0.574,
-      "grad_norm": 15.238197765085344,
-      "kl": 1.044921875,
+      "grad_norm": 5.874146240700076,
+      "kl": 0.482421875,
       "learning_rate": 5.123449705004581e-07,
-      "loss": 0.1585,
-      "reward": 0.9687500298023224,
-      "reward_std": 0.08474057167768478,
-      "rewards/tag_count_reward": 0.9687500298023224,
+      "loss": 0.1124,
+      "reward": 0.375,
+      "reward_std": 0.4778681993484497,
+      "rewards/accuracy_reward": 0.375,
       "step": 574
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 395.1458435058594,
+      "completion_length": 1631.5208740234375,
       "epoch": 0.575,
-      "grad_norm": 3.7259115162429315,
-      "kl": 0.2841796875,
+      "grad_norm": 11.24899950375608,
+      "kl": 0.474609375,
       "learning_rate": 5.107799157635538e-07,
-      "loss": 0.0127,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": -0.0005,
+      "reward": 0.3333333358168602,
+      "reward_std": 0.4314185529947281,
+      "rewards/accuracy_reward": 0.3333333358168602,
       "step": 575
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 383.3541717529297,
+      "completion_length": 1782.1041870117188,
       "epoch": 0.576,
-      "grad_norm": 3.6577494997491047,
-      "kl": 0.208984375,
+      "grad_norm": 4.607009962877916,
+      "kl": 0.4365234375,
       "learning_rate": 5.09215338910999e-07,
-      "loss": 0.0099,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0246,
+      "reward": 0.229166679084301,
+      "reward_std": 0.3592497557401657,
+      "rewards/accuracy_reward": 0.229166679084301,
       "step": 576
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 409.72918701171875,
+      "completion_length": 1696.3958740234375,
       "epoch": 0.577,
-      "grad_norm": 3.734336816963088,
-      "kl": 0.22265625,
+      "grad_norm": 3.062603801453861,
+      "kl": 0.3525390625,
       "learning_rate": 5.076512590066685e-07,
-      "loss": 0.0127,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0555,
+      "reward": 0.0833333358168602,
+      "reward_std": 0.19462472200393677,
+      "rewards/accuracy_reward": 0.0833333358168602,
       "step": 577
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 383.0416717529297,
+      "completion_length": 1811.6458740234375,
       "epoch": 0.578,
-      "grad_norm": 5.111707915314032,
-      "kl": 0.239501953125,
+      "grad_norm": 2.9864086870116227,
+      "kl": 0.2744140625,
       "learning_rate": 5.060876951083828e-07,
-      "loss": 0.0149,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0325,
+      "reward": 0.1041666679084301,
+      "reward_std": 0.26679350435733795,
+      "rewards/accuracy_reward": 0.1041666679084301,
       "step": 578
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 410.5,
+      "completion_length": 1885.8958740234375,
       "epoch": 0.579,
-      "grad_norm": 8.980476325432091,
-      "kl": 0.2880859375,
+      "grad_norm": 6.029997316114094,
+      "kl": 0.22119140625,
       "learning_rate": 5.045246662676741e-07,
-      "loss": 0.0689,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0507,
+      "reward": 0.2708333395421505,
+      "reward_std": 0.31381870806217194,
+      "rewards/accuracy_reward": 0.2708333395421505,
       "step": 579
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 391.3125,
+      "completion_length": 1926.9375610351562,
       "epoch": 0.58,
-      "grad_norm": 8.677092482156407,
-      "kl": 0.1748046875,
+      "grad_norm": 3.2302007662725365,
+      "kl": 0.25244140625,
       "learning_rate": 5.02962191529556e-07,
-      "loss": 0.0902,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0125,
+      "reward": 0.12500000558793545,
+      "reward_std": 0.20090095698833466,
+      "rewards/accuracy_reward": 0.12500000558793545,
       "step": 580
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 356.60418701171875,
+      "completion_length": 1962.479248046875,
       "epoch": 0.581,
-      "grad_norm": 0.7969810760307527,
-      "kl": 0.12744140625,
+      "grad_norm": 5.685512430000827,
+      "kl": 0.275390625,
       "learning_rate": 5.014002899322896e-07,
-      "loss": 0.0056,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0697,
+      "reward": 0.1041666679084301,
+      "reward_std": 0.25740431994199753,
+      "rewards/accuracy_reward": 0.1041666679084301,
       "step": 581
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 402.62501525878906,
+      "completion_length": 1859.9375610351562,
       "epoch": 0.582,
-      "grad_norm": 5.255605393691853,
-      "kl": 0.226806640625,
+      "grad_norm": 4.347416124423323,
+      "kl": 0.2939453125,
       "learning_rate": 4.998389805071536e-07,
-      "loss": 0.0216,
-      "reward": 0.984375,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.053,
+      "reward": 0.12500000558793545,
+      "reward_std": 0.20090095698833466,
+      "rewards/accuracy_reward": 0.12500000558793545,
       "step": 582
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 408.0,
+      "completion_length": 1893.0833740234375,
       "epoch": 0.583,
-      "grad_norm": 4.093041562859633,
-      "kl": 0.19970703125,
+      "grad_norm": 5.95073378955075,
+      "kl": 0.3388671875,
       "learning_rate": 4.982782822782101e-07,
-      "loss": 0.0602,
-      "reward": 0.984375,
-      "reward_std": 0.03884884715080261,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.088,
+      "reward": 0.2083333432674408,
+      "reward_std": 0.39552566409111023,
+      "rewards/accuracy_reward": 0.2083333432674408,
       "step": 583
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 395.4791717529297,
+      "completion_length": 1928.9583740234375,
       "epoch": 0.584,
-      "grad_norm": 11.262303957873653,
-      "kl": 0.150390625,
+      "grad_norm": 2.7161492887069447,
+      "kl": 0.287109375,
       "learning_rate": 4.967182142620745e-07,
-      "loss": 0.1275,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0148,
+      "reward": 0.0833333358168602,
+      "reward_std": 0.12309150397777557,
+      "rewards/accuracy_reward": 0.0833333358168602,
       "step": 584
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 410.68751525878906,
+      "completion_length": 1871.7083740234375,
       "epoch": 0.585,
-      "grad_norm": 2.747042042599716,
-      "kl": 0.076904296875,
+      "grad_norm": 5.494893046329305,
+      "kl": 0.4677734375,
       "learning_rate": 4.951587954676837e-07,
-      "loss": 0.0212,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": 0.1014,
+      "reward": 0.3333333432674408,
+      "reward_std": 0.4056393951177597,
+      "rewards/accuracy_reward": 0.3333333432674408,
       "step": 585
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 407.6458435058594,
+      "completion_length": 1706.8541870117188,
       "epoch": 0.586,
-      "grad_norm": 3.7904544650583105,
-      "kl": 0.2137451171875,
+      "grad_norm": 7.2033946999214935,
+      "kl": 0.486328125,
       "learning_rate": 4.93600044896063e-07,
-      "loss": 0.0532,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.04865618050098419,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0571,
+      "reward": 0.3125000111758709,
+      "reward_std": 0.31381870806217194,
+      "rewards/accuracy_reward": 0.3125000111758709,
       "step": 586
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 379.5416717529297,
+      "completion_length": 1911.4583740234375,
       "epoch": 0.587,
-      "grad_norm": 5.346241073417882,
-      "kl": 0.27197265625,
+      "grad_norm": 3.7012820152401624,
+      "kl": 0.55078125,
       "learning_rate": 4.920419815400968e-07,
-      "loss": 0.0151,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0653,
+      "reward": 0.2291666679084301,
+      "reward_std": 0.3961612284183502,
+      "rewards/accuracy_reward": 0.2291666679084301,
       "step": 587
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 368.1458435058594,
+      "completion_length": 1760.5833740234375,
       "epoch": 0.588,
-      "grad_norm": 0.3159785441102848,
-      "kl": 0.067626953125,
+      "grad_norm": 4.265748668128537,
+      "kl": 0.4228515625,
       "learning_rate": 4.904846243842949e-07,
-      "loss": 0.0029,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0735,
+      "reward": 0.2916666716337204,
+      "reward_std": 0.364890418946743,
+      "rewards/accuracy_reward": 0.2916666716337204,
       "step": 588
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 419.62501525878906,
+      "completion_length": 1666.9583740234375,
       "epoch": 0.589,
-      "grad_norm": 46.96028559035001,
-      "kl": 0.9996337890625,
+      "grad_norm": 3.718042967428544,
+      "kl": 0.6171875,
       "learning_rate": 4.88927992404563e-07,
-      "loss": 0.0576,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0754,
+      "reward": 0.4583333432674408,
+      "reward_std": 0.3157937675714493,
+      "rewards/accuracy_reward": 0.4583333432674408,
       "step": 589
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 397.50001525878906,
+      "completion_length": 1657.541748046875,
       "epoch": 0.59,
-      "grad_norm": 9.089838613001081,
-      "kl": 0.529296875,
+      "grad_norm": 9.232700125365328,
+      "kl": 0.7109375,
       "learning_rate": 4.873721045679706e-07,
-      "loss": 0.1543,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0779,
+      "reward": 0.5625000298023224,
+      "reward_std": 0.3547166883945465,
+      "rewards/accuracy_reward": 0.5625000298023224,
       "step": 590
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 415.04168701171875,
+      "completion_length": 1695.1458740234375,
       "epoch": 0.591,
-      "grad_norm": 0.7738897102064202,
-      "kl": 0.076904296875,
+      "grad_norm": 6.877102332556054,
+      "kl": 0.72265625,
       "learning_rate": 4.858169798325198e-07,
-      "loss": 0.003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0497,
+      "reward": 0.354166679084301,
+      "reward_std": 0.29257264733314514,
+      "rewards/accuracy_reward": 0.354166679084301,
       "step": 591
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 363.68751525878906,
+      "completion_length": 1756.375,
       "epoch": 0.592,
-      "grad_norm": 0.25596770540476016,
-      "kl": 0.07470703125,
+      "grad_norm": 4.313038080960948,
+      "kl": 0.607421875,
       "learning_rate": 4.842626371469149e-07,
-      "loss": 0.003,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0771,
+      "reward": 0.4583333358168602,
+      "reward_std": 0.41502857208251953,
+      "rewards/accuracy_reward": 0.4583333358168602,
       "step": 592
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 417.7083435058594,
+      "completion_length": 1742.2708740234375,
       "epoch": 0.593,
-      "grad_norm": 3.0302408863154127,
-      "kl": 0.30126953125,
+      "grad_norm": 6.335500478021944,
+      "kl": 0.580078125,
       "learning_rate": 4.827090954503308e-07,
-      "loss": 0.0114,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.038,
+      "reward": 0.3958333358168602,
+      "reward_std": 0.4370592087507248,
+      "rewards/accuracy_reward": 0.3958333358168602,
       "step": 593
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 453.8333435058594,
+      "completion_length": 1753.7083740234375,
       "epoch": 0.594,
-      "grad_norm": 4.801912575140599,
-      "kl": 0.26025390625,
+      "grad_norm": 5.087723385957405,
+      "kl": 0.5,
       "learning_rate": 4.811563736721829e-07,
-      "loss": 0.0579,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.07845468074083328,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.0793,
+      "reward": 0.2916666716337204,
+      "reward_std": 0.3234458789229393,
+      "rewards/accuracy_reward": 0.2916666716337204,
       "step": 594
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 461.93751525878906,
+      "completion_length": 1775.5625610351562,
       "epoch": 0.595,
-      "grad_norm": 7.675974503977892,
-      "kl": 0.2900390625,
+      "grad_norm": 5.007007453961451,
+      "kl": 0.759765625,
       "learning_rate": 4.79604490731896e-07,
-      "loss": 0.0444,
-      "reward": 0.9739583730697632,
-      "reward_std": 0.09021097794175148,
-      "rewards/tag_count_reward": 0.9739583730697632,
+      "loss": 0.0356,
+      "reward": 0.2916666865348816,
+      "reward_std": 0.3648904263973236,
+      "rewards/accuracy_reward": 0.2916666865348816,
       "step": 595
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 378.12501525878906,
+      "completion_length": 1845.6875610351562,
       "epoch": 0.596,
-      "grad_norm": 0.15133686111386602,
-      "kl": 0.0579833984375,
+      "grad_norm": 4.69296357302523,
+      "kl": 0.66796875,
       "learning_rate": 4.780534655386743e-07,
-      "loss": 0.0024,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0444,
+      "reward": 0.2291666716337204,
+      "reward_std": 0.4156641364097595,
+      "rewards/accuracy_reward": 0.2291666716337204,
       "step": 596
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 429.2083435058594,
+      "completion_length": 1853.0625610351562,
       "epoch": 0.597,
-      "grad_norm": 3.795833682022691,
-      "kl": 0.302734375,
+      "grad_norm": 6.259908180253775,
+      "kl": 0.5595703125,
       "learning_rate": 4.7650331699127013e-07,
-      "loss": 0.1003,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.04865618795156479,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0648,
+      "reward": 0.35416667722165585,
+      "reward_std": 0.3000393956899643,
+      "rewards/accuracy_reward": 0.35416667722165585,
       "step": 597
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 367.625,
+      "completion_length": 1938.2291870117188,
       "epoch": 0.598,
-      "grad_norm": 3.9422683198568147,
-      "kl": 0.3291015625,
+      "grad_norm": 3.4048598520876965,
+      "kl": 0.6123046875,
       "learning_rate": 4.749540639777539e-07,
-      "loss": 0.043,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0318,
+      "reward": 0.1666666716337204,
+      "reward_std": 0.3647414296865463,
+      "rewards/accuracy_reward": 0.1666666716337204,
       "step": 598
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 438.93751525878906,
+      "completion_length": 1730.8541870117188,
       "epoch": 0.599,
-      "grad_norm": 2.2094625444727685,
-      "kl": 0.19482421875,
+      "grad_norm": 7.674227690378787,
+      "kl": 0.68359375,
       "learning_rate": 4.7340572537528547e-07,
-      "loss": 0.0083,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1016,
+      "reward": 0.3333333432674408,
+      "reward_std": 0.46966953575611115,
+      "rewards/accuracy_reward": 0.3333333432674408,
       "step": 599
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 390.87501525878906,
+      "completion_length": 1710.1458740234375,
       "epoch": 0.6,
-      "grad_norm": 0.9972729819698383,
-      "kl": 0.120849609375,
+      "grad_norm": 4.609109011079948,
+      "kl": 0.7109375,
       "learning_rate": 4.7185832004988133e-07,
-      "loss": 0.0059,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0315,
+      "reward": 0.375,
+      "reward_std": 0.4778681993484497,
+      "rewards/accuracy_reward": 0.375,
       "step": 600
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 360.4791717529297,
+      "completion_length": 1682.5625610351562,
       "epoch": 0.601,
-      "grad_norm": 1.4942812379211767,
-      "kl": 0.1241455078125,
+      "grad_norm": 7.089585364234495,
+      "kl": 0.98046875,
       "learning_rate": 4.703118668561875e-07,
-      "loss": -0.009,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.036,
+      "reward": 0.31250002048909664,
+      "reward_std": 0.29257263243198395,
+      "rewards/accuracy_reward": 0.31250002048909664,
       "step": 601
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 461.2083435058594,
+      "completion_length": 1672.979248046875,
       "epoch": 0.602,
-      "grad_norm": 0.1762287472453783,
-      "kl": 0.062744140625,
+      "grad_norm": 9.63111841145383,
+      "kl": 0.9140625,
       "learning_rate": 4.68766384637248e-07,
-      "loss": 0.0026,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0543,
+      "reward": 0.5625000298023224,
+      "reward_std": 0.38988499343395233,
+      "rewards/accuracy_reward": 0.5625000298023224,
       "step": 602
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 387.6875,
+      "completion_length": 1763.5000610351562,
       "epoch": 0.603,
-      "grad_norm": 2.6832838261921457,
-      "kl": 0.10107421875,
+      "grad_norm": 2.960133561210736,
+      "kl": 0.9296875,
       "learning_rate": 4.672218922242759e-07,
-      "loss": 0.0191,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0326,
+      "reward": 0.12500000558793545,
+      "reward_std": 0.20090095698833466,
+      "rewards/accuracy_reward": 0.12500000558793545,
       "step": 603
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 433.1875,
+      "completion_length": 1576.416748046875,
       "epoch": 0.604,
-      "grad_norm": 3.2447151229056708,
-      "kl": 0.21142578125,
+      "grad_norm": 5.231476487536769,
+      "kl": 0.92578125,
       "learning_rate": 4.656784084364238e-07,
-      "loss": -0.0232,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.07216878235340118,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": -0.0146,
+      "reward": 0.06250000186264515,
+      "reward_std": 0.21650634706020355,
+      "rewards/accuracy_reward": 0.06250000186264515,
       "step": 604
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 389.87501525878906,
+      "completion_length": 1728.0625610351562,
       "epoch": 0.605,
-      "grad_norm": 0.13605051880246277,
-      "kl": 0.0572509765625,
+      "grad_norm": 5.005279150678156,
+      "kl": 1.29296875,
       "learning_rate": 4.641359520805548e-07,
-      "loss": 0.0023,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0631,
+      "reward": 0.2708333395421505,
+      "reward_std": 0.21650634706020355,
+      "rewards/accuracy_reward": 0.2708333395421505,
       "step": 605
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 392.25001525878906,
+      "completion_length": 1713.8541870117188,
       "epoch": 0.606,
-      "grad_norm": 5.4369335865443364,
-      "kl": 0.132080078125,
+      "grad_norm": 8.664849410613432,
+      "kl": 1.4921875,
       "learning_rate": 4.6259454195101267e-07,
-      "loss": 0.0267,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.07216878235340118,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0811,
+      "reward": 0.3958333432674408,
+      "reward_std": 0.29257264733314514,
+      "rewards/accuracy_reward": 0.3958333432674408,
       "step": 606
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 418.87501525878906,
+      "completion_length": 1776.7708740234375,
       "epoch": 0.607,
-      "grad_norm": 3.092282569169051,
-      "kl": 0.146484375,
+      "grad_norm": 2.82922191356062,
+      "kl": 0.80078125,
       "learning_rate": 4.6105419682939316e-07,
-      "loss": 0.0541,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0357,
+      "reward": 0.1875,
+      "reward_std": 0.19526028633117676,
+      "rewards/accuracy_reward": 0.1875,
       "step": 607
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 423.8958435058594,
+      "completion_length": 1893.9166870117188,
       "epoch": 0.608,
-      "grad_norm": 0.24882707008371582,
-      "kl": 0.0673828125,
+      "grad_norm": 5.3567900237368855,
+      "kl": 0.82421875,
       "learning_rate": 4.59514935484316e-07,
-      "loss": 0.0028,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0512,
+      "reward": 0.1250000037252903,
+      "reward_std": 0.3295731097459793,
+      "rewards/accuracy_reward": 0.1250000037252903,
       "step": 608
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 491.43751525878906,
+      "completion_length": 1866.2708740234375,
       "epoch": 0.609,
-      "grad_norm": 6.451615477059651,
-      "kl": 0.456298828125,
+      "grad_norm": 3.5325333096905767,
+      "kl": 0.78125,
       "learning_rate": 4.579767766711944e-07,
-      "loss": 0.1002,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0597,
+      "reward": 0.3333333432674408,
+      "reward_std": 0.3492250069975853,
+      "rewards/accuracy_reward": 0.3333333432674408,
       "step": 609
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 403.50001525878906,
+      "completion_length": 1825.0833740234375,
       "epoch": 0.61,
-      "grad_norm": 0.3635257731764919,
-      "kl": 0.0567626953125,
+      "grad_norm": 7.518171017928333,
+      "kl": 0.958984375,
       "learning_rate": 4.5643973913200837e-07,
-      "loss": 0.0022,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0585,
+      "reward": 0.1458333395421505,
+      "reward_std": 0.22604455053806305,
+      "rewards/accuracy_reward": 0.1458333395421505,
       "step": 610
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 409.0416717529297,
+      "completion_length": 2036.1666870117188,
       "epoch": 0.611,
-      "grad_norm": 12.337782777253606,
-      "kl": 0.49267578125,
+      "grad_norm": 1.6897527311265033,
+      "kl": 1.12109375,
       "learning_rate": 4.549038415950751e-07,
-      "loss": 0.0244,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0498,
+      "reward": 0.0625,
+      "reward_std": 0.11306675523519516,
+      "rewards/accuracy_reward": 0.0625,
       "step": 611
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 408.3333435058594,
+      "completion_length": 2035.5000610351562,
       "epoch": 0.612,
-      "grad_norm": 0.7775751820487108,
-      "kl": 0.0877685546875,
+      "grad_norm": 3.9789113918794805,
+      "kl": 0.953125,
       "learning_rate": 4.5336910277482155e-07,
-      "loss": 0.0035,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0385,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.09731236100196838,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 612
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 437.9375,
+      "completion_length": 2043.4791870117188,
       "epoch": 0.613,
-      "grad_norm": 1.510417692801011,
-      "kl": 0.114013671875,
+      "grad_norm": 1.2446978404478806,
+      "kl": 1.041015625,
       "learning_rate": 4.51835541371556e-07,
-      "loss": 0.0048,
-      "reward": 1.0,
+      "loss": 0.0416,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 613
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 440.04168701171875,
+      "completion_length": 2048.0,
       "epoch": 0.614,
-      "grad_norm": 8.22785879406297,
-      "kl": 0.276611328125,
+      "grad_norm": 1.341717768392871,
+      "kl": 1.06640625,
       "learning_rate": 4.503031760712397e-07,
-      "loss": 0.027,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0427,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 614
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 411.4583435058594,
+      "completion_length": 2048.0,
       "epoch": 0.615,
-      "grad_norm": 0.650664392596077,
-      "kl": 0.07958984375,
+      "grad_norm": 0.27715907440662957,
+      "kl": 0.89453125,
       "learning_rate": 4.4877202554526084e-07,
-      "loss": 0.0034,
-      "reward": 1.0,
+      "loss": 0.0358,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 615
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 401.1458435058594,
+      "completion_length": 2048.0,
       "epoch": 0.616,
-      "grad_norm": 13.66748173578044,
-      "kl": 0.49755859375,
+      "grad_norm": 0.7389666909111163,
+      "kl": 0.861328125,
       "learning_rate": 4.4724210845020494e-07,
-      "loss": 0.0807,
-      "reward": 0.9895833730697632,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833730697632,
+      "loss": 0.0345,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 616
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 402.9583435058594,
+      "completion_length": 2011.1875,
       "epoch": 0.617,
-      "grad_norm": 2.654318753149108,
-      "kl": 0.2138671875,
+      "grad_norm": 0.79534449969503,
+      "kl": 0.78125,
       "learning_rate": 4.457134434276293e-07,
-      "loss": 0.0104,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0318,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 617
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 427.2083435058594,
+      "completion_length": 2048.0,
       "epoch": 0.618,
-      "grad_norm": 2.214910708774484,
-      "kl": 0.2333984375,
+      "grad_norm": 0.7141893890341653,
+      "kl": 0.73046875,
       "learning_rate": 4.441860491038345e-07,
-      "loss": 0.0099,
-      "reward": 1.0,
+      "loss": 0.0292,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 618
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 470.1458435058594,
+      "completion_length": 2048.0,
       "epoch": 0.619,
-      "grad_norm": 1.0104438176626016,
-      "kl": 0.1396484375,
+      "grad_norm": 0.805166956554634,
+      "kl": 0.51953125,
       "learning_rate": 4.4265994408963867e-07,
-      "loss": 0.0066,
-      "reward": 1.0,
+      "loss": 0.0208,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 619
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 422.62501525878906,
+      "completion_length": 2015.3125,
       "epoch": 0.62,
-      "grad_norm": 0.7722857506341341,
-      "kl": 0.1124267578125,
+      "grad_norm": 1.9004221636773446,
+      "kl": 0.4033203125,
       "learning_rate": 4.4113514698014953e-07,
-      "loss": 0.0045,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0673,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 620
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 459.2291717529297,
+      "completion_length": 2048.0,
       "epoch": 0.621,
-      "grad_norm": 0.199694627883351,
-      "kl": 0.0582275390625,
+      "grad_norm": 0.5598422030806369,
+      "kl": 0.4013671875,
       "learning_rate": 4.3961167635453876e-07,
-      "loss": 0.0026,
-      "reward": 1.0,
+      "loss": 0.0161,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 621
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 441.2291717529297,
+      "completion_length": 2048.0,
       "epoch": 0.622,
-      "grad_norm": 2.593170125654589,
-      "kl": 0.0628662109375,
+      "grad_norm": 0.49934924578271644,
+      "kl": 0.3359375,
       "learning_rate": 4.3808955077581546e-07,
-      "loss": -0.0074,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.04237028583884239,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": 0.0135,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 622
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 421.2291717529297,
+      "completion_length": 2040.3958740234375,
       "epoch": 0.623,
-      "grad_norm": 0.28650814113330986,
-      "kl": 0.0794677734375,
+      "grad_norm": 0.5549934402146295,
+      "kl": 0.39453125,
       "learning_rate": 4.365687887905988e-07,
-      "loss": 0.0035,
-      "reward": 1.0,
+      "loss": 0.0158,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 623
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 429.41668701171875,
+      "completion_length": 2046.2708740234375,
       "epoch": 0.624,
-      "grad_norm": 0.15751595560674495,
-      "kl": 0.0628662109375,
+      "grad_norm": 0.9483971308694097,
+      "kl": 0.474609375,
       "learning_rate": 4.350494089288943e-07,
-      "loss": 0.0026,
-      "reward": 1.0,
+      "loss": 0.019,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 624
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 429.2708435058594,
+      "completion_length": 2047.5625,
       "epoch": 0.625,
-      "grad_norm": 2.4315463556233707,
-      "kl": 0.0986328125,
+      "grad_norm": 1.6613305522000854,
+      "kl": 0.3525390625,
       "learning_rate": 4.3353142970386557e-07,
-      "loss": -0.0122,
-      "reward": 0.984375,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0147,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 625
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 445.0,
+      "completion_length": 2009.0833740234375,
       "epoch": 0.626,
-      "grad_norm": 3.0278010700726425,
-      "kl": 0.112548828125,
+      "grad_norm": 2.284848338506896,
+      "kl": 0.2705078125,
       "learning_rate": 4.3201486961161093e-07,
-      "loss": 0.0305,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0039,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 626
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 484.6458435058594,
+      "completion_length": 2036.6041870117188,
       "epoch": 0.627,
-      "grad_norm": 4.435038587352279,
-      "kl": 0.173828125,
+      "grad_norm": 4.536897121823954,
+      "kl": 0.37548828125,
       "learning_rate": 4.304997471309361e-07,
-      "loss": 0.0223,
-      "reward": 0.9635416865348816,
-      "reward_std": 0.1110176332294941,
-      "rewards/tag_count_reward": 0.9635416865348816,
+      "loss": 0.0152,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 627
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 429.66668701171875,
+      "completion_length": 2015.75,
       "epoch": 0.628,
-      "grad_norm": 1.6392048454495811,
-      "kl": 0.093017578125,
+      "grad_norm": 0.9309069905910956,
+      "kl": 0.408203125,
       "learning_rate": 4.2898608072313045e-07,
-      "loss": 0.0047,
-      "reward": 0.984375,
-      "reward_std": 0.03884884715080261,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0164,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 628
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 447.2083435058594,
+      "completion_length": 2006.8958740234375,
       "epoch": 0.629,
-      "grad_norm": 5.466422979992953,
-      "kl": 0.19580078125,
+      "grad_norm": 3.319213329433584,
+      "kl": 0.2451171875,
       "learning_rate": 4.2747388883174154e-07,
-      "loss": 0.0774,
-      "reward": 0.984375,
-      "reward_std": 0.04237028583884239,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0672,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 629
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 435.5833435058594,
+      "completion_length": 2024.2083740234375,
       "epoch": 0.63,
-      "grad_norm": 2.9526648613508013,
-      "kl": 0.156982421875,
+      "grad_norm": 0.7407047908411761,
+      "kl": 0.224609375,
       "learning_rate": 4.2596318988235037e-07,
-      "loss": 0.036,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.046308884397149086,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.009,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 630
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 373.5416717529297,
+      "completion_length": 2009.5208740234375,
       "epoch": 0.631,
-      "grad_norm": 2.932328755604553,
-      "kl": 0.33056640625,
+      "grad_norm": 3.4420534174108646,
+      "kl": 0.18505859375,
       "learning_rate": 4.2445400228234687e-07,
-      "loss": 0.0355,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.024328090250492096,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.029,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.09731236100196838,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 631
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 416.7708435058594,
+      "completion_length": 1968.2083740234375,
       "epoch": 0.632,
-      "grad_norm": 0.34680124590011835,
-      "kl": 0.086181640625,
+      "grad_norm": 5.762857224064397,
+      "kl": 0.16650390625,
       "learning_rate": 4.2294634442070553e-07,
-      "loss": 0.0039,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0939,
+      "reward": 0.06250000186264515,
+      "reward_std": 0.16948115825653076,
+      "rewards/accuracy_reward": 0.06250000186264515,
       "step": 632
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 479.60418701171875,
+      "completion_length": 1986.7291870117188,
       "epoch": 0.633,
-      "grad_norm": 0.40169847681205495,
-      "kl": 0.104736328125,
+      "grad_norm": 5.564237881796581,
+      "kl": 0.1943359375,
       "learning_rate": 4.214402346677619e-07,
-      "loss": 0.0042,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0127,
+      "reward": 0.20833333395421505,
+      "reward_std": 0.3157937675714493,
+      "rewards/accuracy_reward": 0.20833333395421505,
       "step": 633
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 449.6666717529297,
+      "completion_length": 1957.7916870117188,
       "epoch": 0.634,
-      "grad_norm": 1.8717955177940753,
-      "kl": 0.1416015625,
+      "grad_norm": 5.515529400707121,
+      "kl": 0.24462890625,
       "learning_rate": 4.1993569137498776e-07,
-      "loss": 0.0287,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0556,
+      "reward": 0.1666666716337204,
+      "reward_std": 0.3647414296865463,
+      "rewards/accuracy_reward": 0.1666666716337204,
       "step": 634
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 398.125,
+      "completion_length": 2020.2083740234375,
       "epoch": 0.635,
-      "grad_norm": 0.14764771310944494,
-      "kl": 0.0518798828125,
+      "grad_norm": 3.2437612160124965,
+      "kl": 0.19384765625,
       "learning_rate": 4.1843273287476854e-07,
-      "loss": 0.0022,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0066,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 635
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 445.6875,
+      "completion_length": 1943.8541870117188,
       "epoch": 0.636,
-      "grad_norm": 1.8274048455196004,
-      "kl": 0.2001953125,
+      "grad_norm": 5.608684388213837,
+      "kl": 0.19677734375,
       "learning_rate": 4.1693137748017915e-07,
-      "loss": 0.0041,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0709,
+      "reward": 0.2083333432674408,
+      "reward_std": 0.3905205726623535,
+      "rewards/accuracy_reward": 0.2083333432674408,
       "step": 636
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 435.9166717529297,
+      "completion_length": 1945.5833740234375,
       "epoch": 0.637,
-      "grad_norm": 0.2659317168517705,
-      "kl": 0.08740234375,
+      "grad_norm": 7.814902212086977,
+      "kl": 0.28759765625,
       "learning_rate": 4.15431643484761e-07,
-      "loss": 0.0032,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1005,
+      "reward": 0.0833333358168602,
+      "reward_std": 0.24164994060993195,
+      "rewards/accuracy_reward": 0.0833333358168602,
       "step": 637
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 440.4166717529297,
+      "completion_length": 1956.0208740234375,
       "epoch": 0.638,
-      "grad_norm": 3.071094295279427,
-      "kl": 0.24169921875,
+      "grad_norm": 5.120012617480854,
+      "kl": 0.21435546875,
       "learning_rate": 4.1393354916230005e-07,
-      "loss": 0.0487,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": 0.0472,
+      "reward": 0.3541666716337204,
+      "reward_std": 0.36671648919582367,
+      "rewards/accuracy_reward": 0.3541666716337204,
       "step": 638
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 418.4583435058594,
+      "completion_length": 1925.7291870117188,
       "epoch": 0.639,
-      "grad_norm": 7.3329075619624415,
-      "kl": 0.33447265625,
+      "grad_norm": 5.428950835732972,
+      "kl": 0.21923828125,
       "learning_rate": 4.124371127666024e-07,
-      "loss": 0.0151,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": -0.0012,
+      "reward": 0.2500000074505806,
+      "reward_std": 0.41310615837574005,
+      "rewards/accuracy_reward": 0.2500000074505806,
       "step": 639
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 514.0625305175781,
+      "completion_length": 1758.7083740234375,
       "epoch": 0.64,
-      "grad_norm": 3.3242478064876124,
-      "kl": 0.24560546875,
+      "grad_norm": 6.043652759320239,
+      "kl": 0.2216796875,
       "learning_rate": 4.1094235253127374e-07,
-      "loss": 0.0392,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0749,
+      "reward": 0.5000000298023224,
+      "reward_std": 0.3547767102718353,
+      "rewards/accuracy_reward": 0.5000000298023224,
       "step": 640
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 454.9583435058594,
+      "completion_length": 1915.5208740234375,
       "epoch": 0.641,
-      "grad_norm": 3.1850444453646687,
-      "kl": 0.20361328125,
+      "grad_norm": 7.775418435726077,
+      "kl": 0.337890625,
       "learning_rate": 4.0944928666949527e-07,
-      "loss": 0.0431,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": 0.0171,
+      "reward": 0.20833333395421505,
+      "reward_std": 0.29821331799030304,
+      "rewards/accuracy_reward": 0.20833333395421505,
       "step": 641
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 436.81251525878906,
+      "completion_length": 1959.1041870117188,
       "epoch": 0.642,
-      "grad_norm": 4.125701151939386,
-      "kl": 0.35693359375,
+      "grad_norm": 7.21676538133664,
+      "kl": 0.3046875,
       "learning_rate": 4.079579333738039e-07,
-      "loss": 0.1334,
-      "reward": 0.9739583730697632,
-      "reward_std": 0.09021097794175148,
-      "rewards/tag_count_reward": 0.9739583730697632,
+      "loss": 0.0193,
+      "reward": 0.2083333432674408,
+      "reward_std": 0.39552566409111023,
+      "rewards/accuracy_reward": 0.2083333432674408,
       "step": 642
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 460.1041717529297,
+      "completion_length": 1856.6250610351562,
       "epoch": 0.643,
-      "grad_norm": 2.7286225905496653,
-      "kl": 0.12744140625,
+      "grad_norm": 6.352149520865179,
+      "kl": 0.376953125,
       "learning_rate": 4.064683108158685e-07,
-      "loss": 0.0701,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0732,
+      "reward": 0.4375000223517418,
+      "reward_std": 0.4112800657749176,
+      "rewards/accuracy_reward": 0.4375000223517418,
       "step": 643
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 412.87501525878906,
+      "completion_length": 2001.0625610351562,
       "epoch": 0.644,
-      "grad_norm": 10.135554352971921,
-      "kl": 1.005859375,
+      "grad_norm": 7.355988837535293,
+      "kl": 0.310546875,
       "learning_rate": 4.0498043714627006e-07,
-      "loss": 0.038,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0157,
+      "reward": 0.416666679084301,
+      "reward_std": 0.41310615837574005,
+      "rewards/accuracy_reward": 0.416666679084301,
       "step": 644
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 386.5,
+      "completion_length": 1921.3541870117188,
       "epoch": 0.645,
-      "grad_norm": 1.117186995184922,
-      "kl": 0.076904296875,
+      "grad_norm": 7.875404910194463,
+      "kl": 0.279296875,
       "learning_rate": 4.034943304942796e-07,
-      "loss": -0.0208,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0116,
+      "reward": 0.7916666865348816,
+      "reward_std": 0.3722081780433655,
+      "rewards/accuracy_reward": 0.7916666865348816,
       "step": 645
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 417.50001525878906,
+      "completion_length": 2008.5416870117188,
       "epoch": 0.646,
-      "grad_norm": 1.9492601392707025,
-      "kl": 0.16943359375,
+      "grad_norm": 5.6581531329328785,
+      "kl": 0.2978515625,
       "learning_rate": 4.020100089676376e-07,
-      "loss": 0.0064,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.031,
+      "reward": 0.1666666679084301,
+      "reward_std": 0.2748958021402359,
+      "rewards/accuracy_reward": 0.1666666679084301,
       "step": 646
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 440.2708435058594,
+      "completion_length": 1951.1041870117188,
       "epoch": 0.647,
-      "grad_norm": 0.21179588270881652,
-      "kl": 0.05908203125,
+      "grad_norm": 11.276156626232668,
+      "kl": 0.31640625,
       "learning_rate": 4.005274906523336e-07,
-      "loss": 0.0024,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0045,
+      "reward": 0.1458333395421505,
+      "reward_std": 0.3395978510379791,
+      "rewards/accuracy_reward": 0.1458333395421505,
       "step": 647
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 454.2708435058594,
+      "completion_length": 1913.0833740234375,
       "epoch": 0.648,
-      "grad_norm": 12.073619542101616,
-      "kl": 0.352294921875,
+      "grad_norm": 10.212144402716552,
+      "kl": 0.3037109375,
       "learning_rate": 3.9904679361238526e-07,
-      "loss": 0.017,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": -0.0049,
+      "reward": 0.3541666716337204,
+      "reward_std": 0.4414432942867279,
+      "rewards/accuracy_reward": 0.3541666716337204,
       "step": 648
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 397.60418701171875,
+      "completion_length": 1901.6458740234375,
       "epoch": 0.649,
-      "grad_norm": 1.2740144063670524,
-      "kl": 0.140625,
+      "grad_norm": 8.432127628321728,
+      "kl": 0.25634765625,
       "learning_rate": 3.975679358896189e-07,
-      "loss": 0.0061,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0252,
+      "reward": 0.4166666716337204,
+      "reward_std": 0.4314185380935669,
+      "rewards/accuracy_reward": 0.4166666716337204,
       "step": 649
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 430.1666717529297,
+      "completion_length": 1933.6875610351562,
       "epoch": 0.65,
-      "grad_norm": 3.3652514648254273,
-      "kl": 0.37109375,
+      "grad_norm": 9.526091073029102,
+      "kl": 0.24267578125,
       "learning_rate": 3.9609093550344907e-07,
-      "loss": 0.0209,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0671,
+      "reward": 0.3958333432674408,
+      "reward_std": 0.36671648919582367,
+      "rewards/accuracy_reward": 0.3958333432674408,
       "step": 650
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 371.1666717529297,
+      "completion_length": 2022.1875,
       "epoch": 0.651,
-      "grad_norm": 6.504464272319793,
-      "kl": 0.677734375,
+      "grad_norm": 5.294008347733481,
+      "kl": 0.26220703125,
       "learning_rate": 3.946158104506594e-07,
-      "loss": -0.0297,
-      "reward": 0.9739583730697632,
-      "reward_std": 0.09021097794175148,
-      "rewards/tag_count_reward": 0.9739583730697632,
+      "loss": 0.0105,
+      "reward": 0.3541666865348816,
+      "reward_std": 0.3139677196741104,
+      "rewards/accuracy_reward": 0.3541666865348816,
       "step": 651
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 410.3958435058594,
+      "completion_length": 2048.0,
       "epoch": 0.652,
-      "grad_norm": 13.816051573602309,
-      "kl": 0.38525390625,
+      "grad_norm": 5.336035785235213,
+      "kl": 0.3046875,
       "learning_rate": 3.931425787051832e-07,
-      "loss": 0.0443,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0122,
+      "reward": 0.2916666865348816,
+      "reward_std": 0.3314591944217682,
+      "rewards/accuracy_reward": 0.2916666865348816,
       "step": 652
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 434.5833435058594,
+      "completion_length": 1971.4583740234375,
       "epoch": 0.653,
-      "grad_norm": 13.428180365575258,
-      "kl": 1.0625,
+      "grad_norm": 6.644640249176056,
+      "kl": 0.2705078125,
       "learning_rate": 3.9167125821788416e-07,
-      "loss": 0.1626,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.07216878235340118,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.023,
+      "reward": 0.4791666679084301,
+      "reward_std": 0.26679350435733795,
+      "rewards/accuracy_reward": 0.4791666679084301,
       "step": 653
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 444.7708435058594,
+      "completion_length": 2032.0000610351562,
       "epoch": 0.654,
-      "grad_norm": 3.7340291643285846,
-      "kl": 0.38525390625,
+      "grad_norm": 8.477293726839452,
+      "kl": 0.2978515625,
       "learning_rate": 3.902018669163384e-07,
-      "loss": 0.0162,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.0749332383275032,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.01,
+      "reward": 0.25,
+      "reward_std": 0.3647414296865463,
+      "rewards/accuracy_reward": 0.25,
       "step": 654
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 458.60418701171875,
+      "completion_length": 2038.0625,
       "epoch": 0.655,
-      "grad_norm": 0.1697259953337305,
-      "kl": 0.04541015625,
+      "grad_norm": 6.474998773904463,
+      "kl": 0.4169921875,
       "learning_rate": 3.8873442270461485e-07,
-      "loss": 0.0018,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0178,
+      "reward": 0.2500000111758709,
+      "reward_std": 0.19462473690509796,
+      "rewards/accuracy_reward": 0.2500000111758709,
       "step": 655
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 389.7708435058594,
+      "completion_length": 1982.9375610351562,
       "epoch": 0.656,
-      "grad_norm": 0.8838722155559678,
-      "kl": 0.07080078125,
+      "grad_norm": 2.5276637593264684,
+      "kl": 0.34375,
       "learning_rate": 3.872689434630585e-07,
-      "loss": 0.0029,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0162,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 656
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 452.8958435058594,
+      "completion_length": 2012.0625,
       "epoch": 0.657,
-      "grad_norm": 2.6003850864220164,
-      "kl": 0.170166015625,
+      "grad_norm": 7.614377718489059,
+      "kl": 0.431640625,
       "learning_rate": 3.8580544704807117e-07,
-      "loss": 0.0084,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0447,
+      "reward": 0.2708333358168602,
+      "reward_std": 0.3804958313703537,
+      "rewards/accuracy_reward": 0.2708333358168602,
       "step": 657
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 448.31251525878906,
+      "completion_length": 2029.3333740234375,
       "epoch": 0.658,
-      "grad_norm": 1.4956454431121218,
-      "kl": 0.13623046875,
+      "grad_norm": 11.745744253845361,
+      "kl": 0.455078125,
       "learning_rate": 3.843439512918949e-07,
-      "loss": 0.0056,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0207,
+      "reward": 0.4166666865348816,
+      "reward_std": 0.2825479060411453,
+      "rewards/accuracy_reward": 0.4166666865348816,
       "step": 658
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 488.9583435058594,
+      "completion_length": 2013.75,
       "epoch": 0.659,
-      "grad_norm": 2.286765300071012,
-      "kl": 0.162109375,
+      "grad_norm": 9.603009829378893,
+      "kl": 0.380859375,
       "learning_rate": 3.8288447400239443e-07,
-      "loss": 0.0211,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": -0.0172,
+      "reward": 0.458333358168602,
+      "reward_std": 0.4314185380935669,
+      "rewards/accuracy_reward": 0.458333358168602,
       "step": 659
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 408.0833435058594,
+      "completion_length": 2023.0,
       "epoch": 0.66,
-      "grad_norm": 0.41200222233589673,
-      "kl": 0.08740234375,
+      "grad_norm": 7.778414400942728,
+      "kl": 0.4140625,
       "learning_rate": 3.8142703296283953e-07,
-      "loss": 0.0034,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.013,
+      "reward": 0.3958333432674408,
+      "reward_std": 0.4521780461072922,
+      "rewards/accuracy_reward": 0.3958333432674408,
       "step": 660
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 492.5833435058594,
+      "completion_length": 2048.0,
       "epoch": 0.661,
-      "grad_norm": 3.0613933390855377,
-      "kl": 0.0755615234375,
+      "grad_norm": 7.829018375600179,
+      "kl": 0.345703125,
       "learning_rate": 3.7997164593168983e-07,
-      "loss": 0.0422,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0139,
+      "reward": 0.3750000149011612,
+      "reward_std": 0.20090095698833466,
+      "rewards/accuracy_reward": 0.3750000149011612,
       "step": 661
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 451.00001525878906,
+      "completion_length": 2041.7916870117188,
       "epoch": 0.662,
-      "grad_norm": 0.4471118658004672,
-      "kl": 0.095458984375,
+      "grad_norm": 6.9545851830176,
+      "kl": 0.5205078125,
       "learning_rate": 3.785183306423767e-07,
-      "loss": 0.0039,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0208,
+      "reward": 0.1250000037252903,
+      "reward_std": 0.2825479060411453,
+      "rewards/accuracy_reward": 0.1250000037252903,
       "step": 662
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 437.5416717529297,
+      "completion_length": 1989.291748046875,
       "epoch": 0.663,
-      "grad_norm": 4.6981177319338325,
-      "kl": 0.140625,
+      "grad_norm": 11.691274911509083,
+      "kl": 0.4208984375,
       "learning_rate": 3.7706710480308835e-07,
-      "loss": 0.0501,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.023,
+      "reward": 0.4375000111758709,
+      "reward_std": 0.3547166883945465,
+      "rewards/accuracy_reward": 0.4375000111758709,
       "step": 663
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 434.29168701171875,
+      "completion_length": 1986.4375,
       "epoch": 0.664,
-      "grad_norm": 3.2404016855128974,
-      "kl": 0.182861328125,
+      "grad_norm": 10.273049806980591,
+      "kl": 0.404296875,
       "learning_rate": 3.7561798609655373e-07,
-      "loss": 0.055,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0036,
+      "reward": 0.625,
+      "reward_std": 0.39552566409111023,
+      "rewards/accuracy_reward": 0.625,
       "step": 664
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 383.31251525878906,
+      "completion_length": 2048.0,
       "epoch": 0.665,
-      "grad_norm": 0.42080049592277263,
-      "kl": 0.08251953125,
+      "grad_norm": 7.300074785767273,
+      "kl": 0.4892578125,
       "learning_rate": 3.7417099217982686e-07,
-      "loss": 0.0036,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0196,
+      "reward": 0.3958333432674408,
+      "reward_std": 0.3000393956899643,
+      "rewards/accuracy_reward": 0.3958333432674408,
       "step": 665
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 446.3541717529297,
+      "completion_length": 2048.0,
       "epoch": 0.666,
-      "grad_norm": 1.2318748164342126,
-      "kl": 0.15087890625,
+      "grad_norm": 6.4806684140929605,
+      "kl": 0.58203125,
       "learning_rate": 3.72726140684072e-07,
-      "loss": 0.0063,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0233,
+      "reward": 0.25,
+      "reward_std": 0.3083270415663719,
+      "rewards/accuracy_reward": 0.25,
       "step": 666
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 468.75001525878906,
+      "completion_length": 2045.3541870117188,
       "epoch": 0.667,
-      "grad_norm": 1.8456503709812673,
-      "kl": 0.1396484375,
+      "grad_norm": 5.608645739825995,
+      "kl": 0.5556640625,
       "learning_rate": 3.712834492143487e-07,
-      "loss": 0.0208,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0214,
+      "reward": 0.3125000149011612,
+      "reward_std": 0.3076914846897125,
+      "rewards/accuracy_reward": 0.3125000149011612,
       "step": 667
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 437.7708435058594,
+      "completion_length": 1988.5,
       "epoch": 0.668,
-      "grad_norm": 6.344832823638316,
-      "kl": 0.118408203125,
+      "grad_norm": 4.405354646925744,
+      "kl": 0.572265625,
       "learning_rate": 3.6984293534939737e-07,
-      "loss": 0.0746,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0395,
+      "reward": 0.20833333395421505,
+      "reward_std": 0.18523553758859634,
+      "rewards/accuracy_reward": 0.20833333395421505,
       "step": 668
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 435.7708435058594,
+      "completion_length": 2048.0,
       "epoch": 0.669,
-      "grad_norm": 1.7198633924214835,
-      "kl": 0.10986328125,
+      "grad_norm": 3.7408246439835557,
+      "kl": 0.6796875,
       "learning_rate": 3.6840461664142444e-07,
-      "loss": -0.0091,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0271,
+      "reward": 0.0833333358168602,
+      "reward_std": 0.24164992570877075,
+      "rewards/accuracy_reward": 0.0833333358168602,
       "step": 669
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 408.87501525878906,
+      "completion_length": 2048.0,
       "epoch": 0.67,
-      "grad_norm": 2.643321718727809,
-      "kl": 0.2587890625,
+      "grad_norm": 4.672406240036291,
+      "kl": 0.671875,
       "learning_rate": 3.6696851061588994e-07,
-      "loss": 0.0438,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.04956009238958359,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.0269,
+      "reward": 0.06250000186264515,
+      "reward_std": 0.21650634706020355,
+      "rewards/accuracy_reward": 0.06250000186264515,
       "step": 670
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 380.93751525878906,
+      "completion_length": 2020.2708740234375,
       "epoch": 0.671,
-      "grad_norm": 3.507375631504847,
-      "kl": 0.302978515625,
+      "grad_norm": 2.3282256094229417,
+      "kl": 0.6484375,
       "learning_rate": 3.655346347712922e-07,
-      "loss": 0.0146,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0196,
+      "reward": 0.06250000186264515,
+      "reward_std": 0.21650634706020355,
+      "rewards/accuracy_reward": 0.06250000186264515,
       "step": 671
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 393.75001525878906,
+      "completion_length": 2014.3541870117188,
       "epoch": 0.672,
-      "grad_norm": 6.501617653915259,
-      "kl": 0.39306640625,
+      "grad_norm": 4.267684758055409,
+      "kl": 0.615234375,
       "learning_rate": 3.641030065789562e-07,
-      "loss": 0.0163,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.05689104273915291,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.06,
+      "reward": 0.1250000037252903,
+      "reward_std": 0.22040386497974396,
+      "rewards/accuracy_reward": 0.1250000037252903,
       "step": 672
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 421.2916717529297,
+      "completion_length": 2048.0,
       "epoch": 0.673,
-      "grad_norm": 4.276393202562739,
-      "kl": 0.5185546875,
+      "grad_norm": 7.620066593464584,
+      "kl": 0.578125,
       "learning_rate": 3.6267364348281946e-07,
-      "loss": 0.0333,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.0749332383275032,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.0231,
+      "reward": 0.25,
+      "reward_std": 0.3083270415663719,
+      "rewards/accuracy_reward": 0.25,
       "step": 673
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 402.5208435058594,
+      "completion_length": 2011.7916870117188,
       "epoch": 0.674,
-      "grad_norm": 694.7450073397962,
-      "kl": 20.8046875,
+      "grad_norm": 2.3614226363221404,
+      "kl": 0.58984375,
       "learning_rate": 3.612465628992203e-07,
-      "loss": 1.1834,
-      "reward": 0.9687500298023224,
-      "reward_std": 0.0929754413664341,
-      "rewards/tag_count_reward": 0.9687500298023224,
+      "loss": 0.0185,
+      "reward": 0.125,
+      "reward_std": 0.22613351047039032,
+      "rewards/accuracy_reward": 0.125,
       "step": 674
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 396.68751525878906,
+      "completion_length": 2044.5416870117188,
       "epoch": 0.675,
-      "grad_norm": 4.273385615664699,
-      "kl": 0.1492919921875,
+      "grad_norm": 3.8589921298110963,
+      "kl": 0.611328125,
       "learning_rate": 3.5982178221668533e-07,
-      "loss": 0.0074,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0234,
+      "reward": 0.2500000111758709,
+      "reward_std": 0.3547767102718353,
+      "rewards/accuracy_reward": 0.2500000111758709,
       "step": 675
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 419.7916717529297,
+      "completion_length": 1965.2916870117188,
       "epoch": 0.676,
-      "grad_norm": 586.7551051861542,
-      "kl": 10.14404296875,
+      "grad_norm": 4.3290365234595605,
+      "kl": 0.533203125,
       "learning_rate": 3.5839931879571725e-07,
-      "loss": 0.7136,
-      "reward": 0.984375,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0159,
+      "reward": 0.3541666865348816,
+      "reward_std": 0.3705310970544815,
+      "rewards/accuracy_reward": 0.3541666865348816,
       "step": 676
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 429.7708435058594,
+      "completion_length": 2048.0,
       "epoch": 0.677,
-      "grad_norm": 8.04671027131853,
-      "kl": 0.5595703125,
+      "grad_norm": 6.207421782485618,
+      "kl": 0.634765625,
       "learning_rate": 3.5697918996858443e-07,
-      "loss": 0.0337,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0254,
+      "reward": 0.2916666716337204,
+      "reward_std": 0.3389623165130615,
+      "rewards/accuracy_reward": 0.2916666716337204,
       "step": 677
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 488.0833435058594,
+      "completion_length": 2016.2916870117188,
       "epoch": 0.678,
-      "grad_norm": 19.4631689907819,
-      "kl": 1.8203125,
+      "grad_norm": 6.54967286830931,
+      "kl": 0.5234375,
       "learning_rate": 3.555614130391079e-07,
-      "loss": 0.1889,
-      "reward": 0.9687500298023224,
-      "reward_std": 0.08474056795239449,
-      "rewards/tag_count_reward": 0.9687500298023224,
+      "loss": 0.0216,
+      "reward": 0.4583333432674408,
+      "reward_std": 0.4314185380935669,
+      "rewards/accuracy_reward": 0.4583333432674408,
       "step": 678
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 437.9166717529297,
+      "completion_length": 2048.0,
       "epoch": 0.679,
-      "grad_norm": 4.207458753364508,
-      "kl": 0.34423828125,
+      "grad_norm": 5.419619920640218,
+      "kl": 0.5546875,
       "learning_rate": 3.5414600528245266e-07,
-      "loss": 0.0715,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.05689104646444321,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0222,
+      "reward": 0.1041666679084301,
+      "reward_std": 0.25740431994199753,
+      "rewards/accuracy_reward": 0.1041666679084301,
       "step": 679
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 472.22918701171875,
+      "completion_length": 2015.4375,
       "epoch": 0.68,
-      "grad_norm": 8.59331015221391,
-      "kl": 1.037109375,
+      "grad_norm": 5.792517296536558,
+      "kl": 0.494140625,
       "learning_rate": 3.5273298394491515e-07,
-      "loss": 0.0282,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0054,
+      "reward": 0.3541666716337204,
+      "reward_std": 0.4050038456916809,
+      "rewards/accuracy_reward": 0.3541666716337204,
       "step": 680
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 476.2916717529297,
+      "completion_length": 1981.2083740234375,
       "epoch": 0.681,
-      "grad_norm": 9.993205923284181,
-      "kl": 1.244140625,
+      "grad_norm": 6.027246823264619,
+      "kl": 0.4306640625,
       "learning_rate": 3.513223662437147e-07,
-      "loss": 0.2014,
-      "reward": 0.9427083432674408,
-      "reward_std": 0.12026621401309967,
-      "rewards/tag_count_reward": 0.9427083432674408,
+      "loss": 0.0349,
+      "reward": 0.4791666865348816,
+      "reward_std": 0.3547166883945465,
+      "rewards/accuracy_reward": 0.4791666865348816,
       "step": 681
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 453.54168701171875,
+      "completion_length": 2044.6458740234375,
       "epoch": 0.682,
-      "grad_norm": 5.448757627782719,
-      "kl": 0.57421875,
+      "grad_norm": 8.116958065527804,
+      "kl": 0.466796875,
       "learning_rate": 3.4991416936678276e-07,
-      "loss": 0.1092,
-      "reward": 0.984375,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0177,
+      "reward": 0.291666679084301,
+      "reward_std": 0.3704710900783539,
+      "rewards/accuracy_reward": 0.291666679084301,
       "step": 682
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 453.25,
+      "completion_length": 1975.2500610351562,
       "epoch": 0.683,
-      "grad_norm": 8.456088400083507,
-      "kl": 0.61328125,
+      "grad_norm": 5.849833793488573,
+      "kl": 0.4716796875,
       "learning_rate": 3.4850841047255364e-07,
-      "loss": 0.0627,
-      "reward": 0.96875,
-      "reward_std": 0.0929754339158535,
-      "rewards/tag_count_reward": 0.96875,
+      "loss": 0.0561,
+      "reward": 0.4375000149011612,
+      "reward_std": 0.3334706127643585,
+      "rewards/accuracy_reward": 0.4375000149011612,
       "step": 683
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 419.87501525878906,
+      "completion_length": 1946.375,
       "epoch": 0.684,
-      "grad_norm": 3.6825779293110537,
-      "kl": 0.357421875,
+      "grad_norm": 14.025181833137536,
+      "kl": 0.41796875,
       "learning_rate": 3.471051066897562e-07,
-      "loss": 0.0295,
-      "reward": 0.984375,
-      "reward_std": 0.04237028956413269,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0789,
+      "reward": 0.4166666716337204,
+      "reward_std": 0.4213048219680786,
+      "rewards/accuracy_reward": 0.4166666716337204,
       "step": 684
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 371.2083435058594,
+      "completion_length": 2048.0,
       "epoch": 0.685,
-      "grad_norm": 10.382180895310823,
-      "kl": 0.57421875,
+      "grad_norm": 11.099562686579352,
+      "kl": 0.59765625,
       "learning_rate": 3.45704275117204e-07,
-      "loss": 0.0874,
-      "reward": 0.9583333432674408,
-      "reward_std": 0.11378209292888641,
-      "rewards/tag_count_reward": 0.9583333432674408,
+      "loss": 0.0239,
+      "reward": 0.4375000149011612,
+      "reward_std": 0.498006671667099,
+      "rewards/accuracy_reward": 0.4375000149011612,
       "step": 685
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 439.50001525878906,
+      "completion_length": 2048.0,
       "epoch": 0.686,
-      "grad_norm": 7.030962041723798,
-      "kl": 0.576171875,
+      "grad_norm": 9.616738965894056,
+      "kl": 0.4560546875,
       "learning_rate": 3.4430593282358777e-07,
-      "loss": 0.0018,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0183,
+      "reward": 0.3958333432674408,
+      "reward_std": 0.3703821152448654,
+      "rewards/accuracy_reward": 0.3958333432674408,
       "step": 686
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 492.4791717529297,
+      "completion_length": 2048.0,
       "epoch": 0.687,
-      "grad_norm": 14.597759514272322,
-      "kl": 1.191650390625,
+      "grad_norm": 9.870398112338325,
+      "kl": 0.55859375,
       "learning_rate": 3.429100968472668e-07,
-      "loss": 0.1023,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0223,
+      "reward": 0.3333333432674408,
+      "reward_std": 0.4213048070669174,
+      "rewards/accuracy_reward": 0.3333333432674408,
       "step": 687
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 416.6666717529297,
+      "completion_length": 1979.1875,
       "epoch": 0.688,
-      "grad_norm": 30.769876265900503,
-      "kl": 2.765625,
+      "grad_norm": 8.786012347920384,
+      "kl": 0.638671875,
       "learning_rate": 3.4151678419606233e-07,
-      "loss": 0.3468,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.07216878235340118,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0356,
+      "reward": 0.6250000149011612,
+      "reward_std": 0.3648904263973236,
+      "rewards/accuracy_reward": 0.6250000149011612,
       "step": 688
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 437.00001525878906,
+      "completion_length": 1982.4583740234375,
       "epoch": 0.689,
-      "grad_norm": 26.098704511067435,
-      "kl": 1.56103515625,
+      "grad_norm": 10.070909787708215,
+      "kl": 0.673828125,
       "learning_rate": 3.4012601184704904e-07,
-      "loss": 0.1296,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0234,
+      "reward": 0.5,
+      "reward_std": 0.38924944400787354,
+      "rewards/accuracy_reward": 0.5,
       "step": 689
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 439.6666717529297,
+      "completion_length": 2007.1250610351562,
       "epoch": 0.69,
-      "grad_norm": 33.33379609157588,
-      "kl": 2.5234375,
+      "grad_norm": 11.20510624830675,
+      "kl": 0.935546875,
       "learning_rate": 3.387377967463493e-07,
-      "loss": 0.2176,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.06041248142719269,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0164,
+      "reward": 0.5208333432674408,
+      "reward_std": 0.44452594220638275,
+      "rewards/accuracy_reward": 0.5208333432674408,
       "step": 690
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 415.8333435058594,
+      "completion_length": 1985.9375,
       "epoch": 0.691,
-      "grad_norm": 5.181330445699268,
-      "kl": 0.529296875,
+      "grad_norm": 9.415440693920617,
+      "kl": 1.26953125,
       "learning_rate": 3.3735215580892575e-07,
-      "loss": 0.0972,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.05689104646444321,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0411,
+      "reward": 0.4166666716337204,
+      "reward_std": 0.3905205726623535,
+      "rewards/accuracy_reward": 0.4166666716337204,
       "step": 691
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 454.2291717529297,
+      "completion_length": 2024.4375,
       "epoch": 0.692,
-      "grad_norm": 5.785792471904993,
-      "kl": 0.380859375,
+      "grad_norm": 12.001708108176876,
+      "kl": 1.1953125,
       "learning_rate": 3.359691059183761e-07,
-      "loss": 0.0734,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0469,
+      "reward": 0.4791666716337204,
+      "reward_std": 0.4307829737663269,
+      "rewards/accuracy_reward": 0.4791666716337204,
       "step": 692
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 442.06251525878906,
+      "completion_length": 1986.7708740234375,
       "epoch": 0.693,
-      "grad_norm": 17.32190836556746,
-      "kl": 1.2958984375,
+      "grad_norm": 10.796682159440087,
+      "kl": 1.0390625,
       "learning_rate": 3.3458866392672694e-07,
-      "loss": 0.0677,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.04070868343114853,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0553,
+      "reward": 0.6041666716337204,
+      "reward_std": 0.3703821003437042,
+      "rewards/accuracy_reward": 0.6041666716337204,
       "step": 693
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 483.12501525878906,
+      "completion_length": 2047.2083740234375,
       "epoch": 0.694,
-      "grad_norm": 3.844960881993056,
-      "kl": 0.6005859375,
+      "grad_norm": 13.984664678064933,
+      "kl": 1.001953125,
       "learning_rate": 3.3321084665422803e-07,
-      "loss": 0.0806,
-      "reward": 0.96875,
-      "reward_std": 0.0625,
-      "rewards/tag_count_reward": 0.96875,
+      "loss": 0.0399,
+      "reward": 0.6041666865348816,
+      "reward_std": 0.4464483857154846,
+      "rewards/accuracy_reward": 0.6041666865348816,
       "step": 694
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 432.50001525878906,
+      "completion_length": 2048.0,
       "epoch": 0.695,
-      "grad_norm": 6.264027723583563,
-      "kl": 0.557861328125,
+      "grad_norm": 6.849750871683827,
+      "kl": 1.0625,
       "learning_rate": 3.3183567088914833e-07,
-      "loss": 0.062,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.04865618050098419,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0426,
+      "reward": 0.2291666716337204,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.2291666716337204,
       "step": 695
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 417.62501525878906,
+      "completion_length": 1996.0833740234375,
       "epoch": 0.696,
-      "grad_norm": 5.851220345933992,
-      "kl": 0.6953125,
+      "grad_norm": 3.4080093190683014,
+      "kl": 0.861328125,
       "learning_rate": 3.3046315338757026e-07,
-      "loss": 0.0707,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0346,
+      "reward": 0.25,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.25,
       "step": 696
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 448.7083435058594,
+      "completion_length": 1942.7708740234375,
       "epoch": 0.697,
-      "grad_norm": 19.52816493353307,
-      "kl": 0.5810546875,
+      "grad_norm": 1.905133084913848,
+      "kl": 0.474609375,
       "learning_rate": 3.290933108731866e-07,
-      "loss": 0.0484,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.04237028583884239,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": 0.019,
+      "reward": 0.75,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.75,
       "step": 697
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 481.1041717529297,
+      "completion_length": 1995.25,
       "epoch": 0.698,
-      "grad_norm": 11.159572266012402,
-      "kl": 1.13671875,
+      "grad_norm": 5.79043214127697,
+      "kl": 0.6484375,
       "learning_rate": 3.2772616003709616e-07,
-      "loss": 0.1515,
-      "reward": 0.9583333730697632,
-      "reward_std": 0.10368667915463448,
-      "rewards/tag_count_reward": 0.9583333730697632,
+      "loss": 0.0856,
+      "reward": 0.22916667722165585,
+      "reward_std": 0.16948114335536957,
+      "rewards/accuracy_reward": 0.22916667722165585,
       "step": 698
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 401.3541717529297,
+      "completion_length": 2048.0,
       "epoch": 0.699,
-      "grad_norm": 11.300893647697604,
-      "kl": 1.216796875,
+      "grad_norm": 12.277679315718517,
+      "kl": 0.748046875,
       "learning_rate": 3.263617175376001e-07,
-      "loss": 0.1522,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.0749332457780838,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.0299,
+      "reward": 0.06250000186264515,
+      "reward_std": 0.21650634706020355,
+      "rewards/accuracy_reward": 0.06250000186264515,
       "step": 699
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 503.20835876464844,
+      "completion_length": 2048.0,
       "epoch": 0.7,
-      "grad_norm": 26.92750609594345,
-      "kl": 1.82568359375,
+      "grad_norm": 7.086249255608723,
+      "kl": 0.513671875,
       "learning_rate": 3.250000000000001e-07,
-      "loss": 0.1622,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.05689104646444321,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0205,
+      "reward": 0.3125,
+      "reward_std": 0.2436249926686287,
+      "rewards/accuracy_reward": 0.3125,
       "step": 700
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 510.06251525878906,
+      "completion_length": 1975.3125610351562,
       "epoch": 0.701,
-      "grad_norm": 10.494983415892529,
-      "kl": 1.27734375,
+      "grad_norm": 4.422576244051579,
+      "kl": 0.455078125,
       "learning_rate": 3.2364102401639423e-07,
-      "loss": 0.1745,
-      "reward": 0.9583333730697632,
-      "reward_std": 0.11039762571454048,
-      "rewards/tag_count_reward": 0.9583333730697632,
+      "loss": 0.0335,
+      "reward": 0.1458333432674408,
+      "reward_std": 0.12873217463493347,
+      "rewards/accuracy_reward": 0.1458333432674408,
       "step": 701
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 446.9583435058594,
+      "completion_length": 2009.4791870117188,
       "epoch": 0.702,
-      "grad_norm": 11.436039976655804,
-      "kl": 1.490234375,
+      "grad_norm": 10.47695831567546,
+      "kl": 0.705078125,
       "learning_rate": 3.222848061454764e-07,
-      "loss": 0.2968,
-      "reward": 0.9687500298023224,
-      "reward_std": 0.08474057167768478,
-      "rewards/tag_count_reward": 0.9687500298023224,
+      "loss": 0.0289,
+      "reward": 0.27083333395421505,
+      "reward_std": 0.2983022928237915,
+      "rewards/accuracy_reward": 0.27083333395421505,
       "step": 702
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 436.54168701171875,
+      "completion_length": 2013.125,
       "epoch": 0.703,
-      "grad_norm": 3.1017789326327474,
-      "kl": 0.48046875,
+      "grad_norm": 6.014755843647907,
+      "kl": 0.53125,
       "learning_rate": 3.209313629123329e-07,
-      "loss": 0.061,
-      "reward": 0.984375,
-      "reward_std": 0.03884884715080261,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.035,
+      "reward": 0.2708333432674408,
+      "reward_std": 0.259290412068367,
+      "rewards/accuracy_reward": 0.2708333432674408,
       "step": 703
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 422.5833435058594,
+      "completion_length": 1973.4166870117188,
       "epoch": 0.704,
-      "grad_norm": 4.996298099273398,
-      "kl": 0.57568359375,
+      "grad_norm": 7.9057735870511845,
+      "kl": 0.4580078125,
       "learning_rate": 3.195807108082429e-07,
-      "loss": 0.0241,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.07216878235340118,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.1002,
+      "reward": 0.2916666716337204,
+      "reward_std": 0.33896228671073914,
+      "rewards/accuracy_reward": 0.2916666716337204,
       "step": 704
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 394.0208435058594,
+      "completion_length": 2016.1875,
       "epoch": 0.705,
-      "grad_norm": 4.677444483256973,
-      "kl": 0.2783203125,
+      "grad_norm": 4.043324321947272,
+      "kl": 0.578125,
       "learning_rate": 3.182328662904756e-07,
-      "loss": 0.0153,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.021,
+      "reward": 0.12500000558793545,
+      "reward_std": 0.20090095698833466,
+      "rewards/accuracy_reward": 0.12500000558793545,
       "step": 705
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 506.2708435058594,
+      "completion_length": 1956.1458740234375,
       "epoch": 0.706,
-      "grad_norm": 5.964023130051764,
-      "kl": 0.71533203125,
+      "grad_norm": 9.527335907644892,
+      "kl": 0.6484375,
       "learning_rate": 3.168878457820915e-07,
-      "loss": 0.0581,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": 0.0467,
+      "reward": 0.1458333395421505,
+      "reward_std": 0.29257264733314514,
+      "rewards/accuracy_reward": 0.1458333395421505,
       "step": 706
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 367.50001525878906,
+      "completion_length": 1973.0,
       "epoch": 0.707,
-      "grad_norm": 4.32921957152037,
-      "kl": 0.15380859375,
+      "grad_norm": 2.7153218499563643,
+      "kl": 0.4765625,
       "learning_rate": 3.155456656717408e-07,
-      "loss": 0.0155,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0733,
+      "reward": 0.1458333432674408,
+      "reward_std": 0.20272701978683472,
+      "rewards/accuracy_reward": 0.1458333432674408,
       "step": 707
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 439.41668701171875,
+      "completion_length": 2011.25,
       "epoch": 0.708,
-      "grad_norm": 6.0390044634387765,
-      "kl": 0.15966796875,
+      "grad_norm": 2.419975793921499,
+      "kl": 0.595703125,
       "learning_rate": 3.142063423134644e-07,
-      "loss": 0.0001,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0436,
+      "reward": 0.1041666716337204,
+      "reward_std": 0.12873217463493347,
+      "rewards/accuracy_reward": 0.1041666716337204,
       "step": 708
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 512.25,
+      "completion_length": 2048.0,
       "epoch": 0.709,
-      "grad_norm": 39.10476230531704,
-      "kl": 1.29296875,
+      "grad_norm": 2.7100430624271237,
+      "kl": 0.57421875,
       "learning_rate": 3.1286989202649503e-07,
-      "loss": 0.5214,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.07216878235340118,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.023,
+      "reward": 0.1041666716337204,
+      "reward_std": 0.12873217463493347,
+      "rewards/accuracy_reward": 0.1041666716337204,
       "step": 709
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 397.6041717529297,
+      "completion_length": 1911.7083740234375,
       "epoch": 0.71,
-      "grad_norm": 1.411237124800069,
-      "kl": 0.1256103515625,
+      "grad_norm": 1.8042488580002678,
+      "kl": 0.619140625,
       "learning_rate": 3.115363310950578e-07,
-      "loss": 0.0068,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.088,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 710
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 382.97918701171875,
+      "completion_length": 1858.1458740234375,
       "epoch": 0.711,
-      "grad_norm": 11.257531925319636,
-      "kl": 1.0078125,
+      "grad_norm": 2.3880876457618956,
+      "kl": 0.580078125,
       "learning_rate": 3.102056757681715e-07,
-      "loss": 0.1469,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0881,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 711
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 459.68751525878906,
+      "completion_length": 1860.3958740234375,
       "epoch": 0.712,
-      "grad_norm": 16.25512877311021,
-      "kl": 1.931640625,
+      "grad_norm": 5.4148894673800605,
+      "kl": 0.3603515625,
       "learning_rate": 3.0887794225945143e-07,
-      "loss": 0.1908,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.0749332457780838,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.0761,
+      "reward": 0.10416666977107525,
+      "reward_std": 0.2574043273925781,
+      "rewards/accuracy_reward": 0.10416666977107525,
       "step": 712
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 470.91668701171875,
+      "completion_length": 1897.2083740234375,
       "epoch": 0.713,
-      "grad_norm": 10.833452938881697,
+      "grad_norm": 4.601634887396135,
       "kl": 0.4189453125,
       "learning_rate": 3.075531467469116e-07,
-      "loss": 0.0206,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0716,
+      "reward": 0.1041666716337204,
+      "reward_std": 0.12873217463493347,
+      "rewards/accuracy_reward": 0.1041666716337204,
       "step": 713
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 464.5208435058594,
+      "completion_length": 1883.6041870117188,
       "epoch": 0.714,
-      "grad_norm": 2.80201483686987,
-      "kl": 0.4462890625,
+      "grad_norm": 4.584186785771121,
+      "kl": 0.3525390625,
       "learning_rate": 3.062313053727671e-07,
-      "loss": 0.0226,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0068,
+      "reward": 0.3125,
+      "reward_std": 0.3258185237646103,
+      "rewards/accuracy_reward": 0.3125,
       "step": 714
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 507.6458435058594,
+      "completion_length": 1866.1458740234375,
       "epoch": 0.715,
-      "grad_norm": 4.41405067914247,
-      "kl": 0.33935546875,
+      "grad_norm": 3.840215708938236,
+      "kl": 0.43359375,
       "learning_rate": 3.0491243424323783e-07,
-      "loss": 0.0414,
-      "reward": 0.984375,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": -0.0292,
+      "reward": 0.06250000186264515,
+      "reward_std": 0.21650634706020355,
+      "rewards/accuracy_reward": 0.06250000186264515,
       "step": 715
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 390.1458435058594,
+      "completion_length": 1906.75,
       "epoch": 0.716,
-      "grad_norm": 3.201216632549719,
-      "kl": 0.20703125,
+      "grad_norm": 6.768707211702417,
+      "kl": 0.2822265625,
       "learning_rate": 3.0359654942835247e-07,
-      "loss": 0.0293,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.1091,
+      "reward": 0.2708333358168602,
+      "reward_std": 0.3804958313703537,
+      "rewards/accuracy_reward": 0.2708333358168602,
       "step": 716
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 460.2708435058594,
+      "completion_length": 1758.1041870117188,
       "epoch": 0.717,
-      "grad_norm": 2.3102229608350924,
-      "kl": 0.1923828125,
+      "grad_norm": 6.6347140759379455,
+      "kl": 0.396484375,
       "learning_rate": 3.02283666961752e-07,
-      "loss": 0.0096,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0851,
+      "reward": 0.3958333432674408,
+      "reward_std": 0.4835088700056076,
+      "rewards/accuracy_reward": 0.3958333432674408,
       "step": 717
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 417.3958435058594,
+      "completion_length": 1940.0208740234375,
       "epoch": 0.718,
-      "grad_norm": 12.582863511927389,
-      "kl": 0.5849609375,
+      "grad_norm": 3.70349150487481,
+      "kl": 0.3056640625,
       "learning_rate": 3.0097380284049523e-07,
-      "loss": 0.0379,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0032,
+      "reward": 0.1875000111758709,
+      "reward_std": 0.3470645844936371,
+      "rewards/accuracy_reward": 0.1875000111758709,
       "step": 718
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 491.47918701171875,
+      "completion_length": 1904.125,
       "epoch": 0.719,
-      "grad_norm": 1.3415707838315272,
-      "kl": 0.21337890625,
+      "grad_norm": 4.973332991849756,
+      "kl": 0.2783203125,
       "learning_rate": 2.996669730248628e-07,
-      "loss": 0.0092,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0587,
+      "reward": 0.3750000149011612,
+      "reward_std": 0.39552566409111023,
+      "rewards/accuracy_reward": 0.3750000149011612,
       "step": 719
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 443.56251525878906,
+      "completion_length": 2016.5625,
       "epoch": 0.72,
-      "grad_norm": 1.999158876101931,
-      "kl": 0.093994140625,
+      "grad_norm": 4.785296547356898,
+      "kl": 0.48828125,
       "learning_rate": 2.9836319343816397e-07,
-      "loss": 0.0036,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0127,
+      "reward": 0.4375,
+      "reward_std": 0.38796254992485046,
+      "rewards/accuracy_reward": 0.4375,
       "step": 720
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 519.0833435058594,
+      "completion_length": 1878.3541870117188,
       "epoch": 0.721,
-      "grad_norm": 16.99046812237398,
-      "kl": 0.5625,
+      "grad_norm": 3.5383124001056934,
+      "kl": 0.369140625,
       "learning_rate": 2.9706247996654134e-07,
-      "loss": 0.1438,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": -0.0199,
+      "reward": 0.31250002048909664,
+      "reward_std": 0.3139677047729492,
+      "rewards/accuracy_reward": 0.31250002048909664,
       "step": 721
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 430.60418701171875,
+      "completion_length": 1863.3541870117188,
       "epoch": 0.722,
-      "grad_norm": 4.075599579072145,
-      "kl": 0.677001953125,
+      "grad_norm": 5.7159107373481,
+      "kl": 0.37890625,
       "learning_rate": 2.9576484845877793e-07,
-      "loss": 0.0711,
-      "reward": 0.984375,
-      "reward_std": 0.03884884715080261,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0286,
+      "reward": 0.520833358168602,
+      "reward_std": 0.5010893195867538,
+      "rewards/accuracy_reward": 0.520833358168602,
       "step": 722
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 506.35418701171875,
+      "completion_length": 1982.5625,
       "epoch": 0.723,
-      "grad_norm": 26.315070501082214,
-      "kl": 1.134765625,
+      "grad_norm": 3.370825366161159,
+      "kl": 0.30859375,
       "learning_rate": 2.944703147261046e-07,
-      "loss": 0.1994,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.06317693740129471,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.0058,
+      "reward": 0.520833358168602,
+      "reward_std": 0.26679350435733795,
+      "rewards/accuracy_reward": 0.520833358168602,
       "step": 723
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 513.4791717529297,
+      "completion_length": 1852.0625,
       "epoch": 0.724,
-      "grad_norm": 16.348844686428375,
-      "kl": 0.98828125,
+      "grad_norm": 11.942828183675445,
+      "kl": 0.3349609375,
       "learning_rate": 2.931788945420058e-07,
-      "loss": 0.2025,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.0749332457780838,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": -0.0965,
+      "reward": 0.4166666865348816,
+      "reward_std": 0.499263271689415,
+      "rewards/accuracy_reward": 0.4166666865348816,
       "step": 724
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 411.5625,
+      "completion_length": 1869.5208740234375,
       "epoch": 0.725,
-      "grad_norm": 0.6207821166682146,
-      "kl": 0.09716796875,
+      "grad_norm": 5.948441251583885,
+      "kl": 0.24658203125,
       "learning_rate": 2.918906036420294e-07,
-      "loss": 0.0043,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0532,
+      "reward": 0.8750000298023224,
+      "reward_std": 0.33896228671073914,
+      "rewards/accuracy_reward": 0.8750000298023224,
       "step": 725
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 483.8333435058594,
+      "completion_length": 2008.8958740234375,
       "epoch": 0.726,
-      "grad_norm": 12.43240779138603,
-      "kl": 0.787109375,
+      "grad_norm": 6.118491110208857,
+      "kl": 0.2607421875,
       "learning_rate": 2.9060545772359305e-07,
-      "loss": 0.1083,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.09021097794175148,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.0198,
+      "reward": 0.8333333432674408,
+      "reward_std": 0.36474141478538513,
+      "rewards/accuracy_reward": 0.8333333432674408,
       "step": 726
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 492.43751525878906,
+      "completion_length": 2013.7708740234375,
       "epoch": 0.727,
-      "grad_norm": 2.1358171479466765,
-      "kl": 0.2100830078125,
+      "grad_norm": 5.222536085143853,
+      "kl": 0.3369140625,
       "learning_rate": 2.893234724457946e-07,
-      "loss": 0.009,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0353,
+      "reward": 0.3541666679084301,
+      "reward_std": 0.32335688173770905,
+      "rewards/accuracy_reward": 0.3541666679084301,
       "step": 727
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 521.7083435058594,
+      "completion_length": 1713.2708740234375,
       "epoch": 0.728,
-      "grad_norm": 12.834481320766775,
-      "kl": 1.7900390625,
+      "grad_norm": 6.030628216826682,
+      "kl": 0.23583984375,
       "learning_rate": 2.8804466342921987e-07,
-      "loss": 0.2872,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.05689104646444321,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0282,
+      "reward": 0.3958333432674408,
+      "reward_std": 0.3973517566919327,
+      "rewards/accuracy_reward": 0.3958333432674408,
       "step": 728
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 493.7708435058594,
+      "completion_length": 1945.5208740234375,
       "epoch": 0.729,
-      "grad_norm": 1.9494794184581465,
-      "kl": 0.25,
+      "grad_norm": 6.901440744321142,
+      "kl": 0.29541015625,
       "learning_rate": 2.86769046255753e-07,
-      "loss": 0.0108,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0712,
+      "reward": 0.5000000149011612,
+      "reward_std": 0.5111140608787537,
+      "rewards/accuracy_reward": 0.5000000149011612,
       "step": 729
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 492.79168701171875,
+      "completion_length": 2010.7291870117188,
       "epoch": 0.73,
-      "grad_norm": 15.846940368073113,
-      "kl": 1.458984375,
+      "grad_norm": 5.848524850620918,
+      "kl": 0.40625,
       "learning_rate": 2.854966364683872e-07,
-      "loss": 0.2833,
-      "reward": 0.984375,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0163,
+      "reward": 0.5208333432674408,
+      "reward_std": 0.3749151676893234,
+      "rewards/accuracy_reward": 0.5208333432674408,
       "step": 730
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 474.08335876464844,
+      "completion_length": 1906.3125,
       "epoch": 0.731,
-      "grad_norm": 7.863889311889127,
-      "kl": 0.3837890625,
+      "grad_norm": 7.62440091127239,
+      "kl": 0.2734375,
       "learning_rate": 2.842274495710335e-07,
-      "loss": 0.0895,
-      "reward": 0.9687500298023224,
-      "reward_std": 0.08474057167768478,
-      "rewards/tag_count_reward": 0.9687500298023224,
+      "loss": 0.0446,
+      "reward": 0.6250000298023224,
+      "reward_std": 0.4622027724981308,
+      "rewards/accuracy_reward": 0.6250000298023224,
       "step": 731
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 378.9166717529297,
+      "completion_length": 1896.7708740234375,
       "epoch": 0.732,
-      "grad_norm": 11.632733691365143,
-      "kl": 1.029296875,
+      "grad_norm": 8.741686227735412,
+      "kl": 0.357421875,
       "learning_rate": 2.829615010283344e-07,
-      "loss": 0.1566,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": -0.0193,
+      "reward": 0.6041666865348816,
+      "reward_std": 0.4112800657749176,
+      "rewards/accuracy_reward": 0.6041666865348816,
       "step": 732
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 441.50001525878906,
+      "completion_length": 2007.8958740234375,
       "epoch": 0.733,
-      "grad_norm": 3.9288806935544667,
-      "kl": 0.15625,
+      "grad_norm": 7.592914736448017,
+      "kl": 0.599609375,
       "learning_rate": 2.8169880626547283e-07,
-      "loss": 0.0414,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0021,
+      "reward": 0.4791666716337204,
+      "reward_std": 0.4156641513109207,
+      "rewards/accuracy_reward": 0.4791666716337204,
       "step": 733
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 448.0208435058594,
+      "completion_length": 1904.2083740234375,
       "epoch": 0.734,
-      "grad_norm": 8.40097124257542,
-      "kl": 0.654296875,
+      "grad_norm": 6.670531461700367,
+      "kl": 0.4130859375,
       "learning_rate": 2.8043938066798645e-07,
-      "loss": 0.1139,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.07216878235340118,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.039,
+      "reward": 0.8541666865348816,
+      "reward_std": 0.29257263243198395,
+      "rewards/accuracy_reward": 0.8541666865348816,
       "step": 734
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 494.95835876464844,
+      "completion_length": 1941.25,
       "epoch": 0.735,
-      "grad_norm": 16.13408884785172,
-      "kl": 1.859375,
+      "grad_norm": 9.332499956202208,
+      "kl": 0.4482421875,
       "learning_rate": 2.791832395815782e-07,
-      "loss": 0.2235,
-      "reward": 0.9895833730697632,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833730697632,
+      "loss": -0.0041,
+      "reward": 0.5000000149011612,
+      "reward_std": 0.4287715405225754,
+      "rewards/accuracy_reward": 0.5000000149011612,
       "step": 735
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 444.8541717529297,
+      "completion_length": 1980.7916870117188,
       "epoch": 0.736,
-      "grad_norm": 1.6994100231244034,
-      "kl": 0.155517578125,
+      "grad_norm": 7.711074323894233,
+      "kl": 0.3642578125,
       "learning_rate": 2.7793039831193133e-07,
-      "loss": 0.0063,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": -0.0045,
+      "reward": 0.5416666716337204,
+      "reward_std": 0.487981915473938,
+      "rewards/accuracy_reward": 0.5416666716337204,
       "step": 736
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 529.7083587646484,
+      "completion_length": 1871.5000610351562,
       "epoch": 0.737,
-      "grad_norm": 28.00915402328376,
-      "kl": 1.4169921875,
+      "grad_norm": 3.6431284396029806,
+      "kl": 0.4794921875,
       "learning_rate": 2.766808721245211e-07,
-      "loss": 0.2558,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.024328090250492096,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0005,
+      "reward": 0.4583333432674408,
+      "reward_std": 0.2536497712135315,
+      "rewards/accuracy_reward": 0.4583333432674408,
       "step": 737
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 464.7083435058594,
+      "completion_length": 1911.1041870117188,
       "epoch": 0.738,
-      "grad_norm": 31.317454674009355,
-      "kl": 1.384765625,
+      "grad_norm": 9.862514363140383,
+      "kl": 0.3095703125,
       "learning_rate": 2.7543467624442956e-07,
-      "loss": 0.3725,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": -0.044,
+      "reward": 0.3541666865348816,
+      "reward_std": 0.47795717418193817,
+      "rewards/accuracy_reward": 0.3541666865348816,
       "step": 738
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 483.41668701171875,
+      "completion_length": 1901.7291870117188,
       "epoch": 0.739,
-      "grad_norm": 25.5966330668809,
-      "kl": 1.373046875,
+      "grad_norm": 4.155814409528474,
+      "kl": 0.416015625,
       "learning_rate": 2.741918258561607e-07,
-      "loss": 0.3158,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": 0.017,
+      "reward": 0.6666666716337204,
+      "reward_std": 0.24179892241954803,
+      "rewards/accuracy_reward": 0.6666666716337204,
       "step": 739
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 427.4791717529297,
+      "completion_length": 2013.4375,
       "epoch": 0.74,
-      "grad_norm": 28.50096014279551,
-      "kl": 2.21484375,
+      "grad_norm": 5.950654893921627,
+      "kl": 0.59765625,
       "learning_rate": 2.729523361034538e-07,
-      "loss": 0.4558,
-      "reward": 0.984375,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0088,
+      "reward": 0.25,
+      "reward_std": 0.4465373605489731,
+      "rewards/accuracy_reward": 0.25,
       "step": 740
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 491.3958435058594,
+      "completion_length": 1979.479248046875,
       "epoch": 0.741,
-      "grad_norm": 8.654662460777226,
-      "kl": 1.49609375,
+      "grad_norm": 5.240726191845124,
+      "kl": 0.4638671875,
       "learning_rate": 2.717162220891007e-07,
-      "loss": 0.2585,
-      "reward": 0.9635416865348816,
-      "reward_std": 0.10278276726603508,
-      "rewards/tag_count_reward": 0.9635416865348816,
+      "loss": 0.0193,
+      "reward": 0.75,
+      "reward_std": 0.22613351047039032,
+      "rewards/accuracy_reward": 0.75,
       "step": 741
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 499.25001525878906,
+      "completion_length": 1953.8541870117188,
       "epoch": 0.742,
-      "grad_norm": 11.857589451422239,
-      "kl": 0.6640625,
+      "grad_norm": 5.832980833418605,
+      "kl": 0.5380859375,
       "learning_rate": 2.7048349887476037e-07,
-      "loss": 0.0695,
-      "reward": 0.984375,
-      "reward_std": 0.03884884715080261,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0269,
+      "reward": 0.5,
+      "reward_std": 0.33896228671073914,
+      "rewards/accuracy_reward": 0.5,
       "step": 742
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 464.0208435058594,
+      "completion_length": 2048.0,
       "epoch": 0.743,
-      "grad_norm": 9.793252984079043,
-      "kl": 0.4365234375,
+      "grad_norm": 4.872062740753898,
+      "kl": 0.681640625,
       "learning_rate": 2.692541814807763e-07,
-      "loss": 0.164,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.024328090250492096,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0273,
+      "reward": 0.1875,
+      "reward_std": 0.3956146538257599,
+      "rewards/accuracy_reward": 0.1875,
       "step": 743
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 485.8958435058594,
+      "completion_length": 2002.0208740234375,
       "epoch": 0.744,
-      "grad_norm": 4.419154289930755,
-      "kl": 0.2177734375,
+      "grad_norm": 9.482384492293637,
+      "kl": 0.455078125,
       "learning_rate": 2.6802828488599294e-07,
-      "loss": 0.0177,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0342,
+      "reward": 0.3750000149011612,
+      "reward_std": 0.49544869363307953,
+      "rewards/accuracy_reward": 0.3750000149011612,
       "step": 744
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 506.64585876464844,
+      "completion_length": 1905.4583740234375,
       "epoch": 0.745,
-      "grad_norm": 10.681726054990179,
-      "kl": 0.51904296875,
+      "grad_norm": 7.345967116586366,
+      "kl": 0.576171875,
       "learning_rate": 2.6680582402757324e-07,
-      "loss": 0.1505,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0199,
+      "reward": 0.2916666716337204,
+      "reward_std": 0.3492250069975853,
+      "rewards/accuracy_reward": 0.2916666716337204,
       "step": 745
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 438.7916717529297,
+      "completion_length": 2030.5,
       "epoch": 0.746,
-      "grad_norm": 5.303740743308652,
-      "kl": 0.40185546875,
+      "grad_norm": 2.8789717557637258,
+      "kl": 0.658203125,
       "learning_rate": 2.655868138008171e-07,
-      "loss": 0.1217,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0217,
+      "reward": 0.06250000186264515,
+      "reward_std": 0.16948114335536957,
+      "rewards/accuracy_reward": 0.06250000186264515,
       "step": 746
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 400.22918701171875,
+      "completion_length": 2000.4375,
       "epoch": 0.747,
-      "grad_norm": 7.818333629793629,
-      "kl": 0.388671875,
+      "grad_norm": 4.174398129470502,
+      "kl": 0.5849609375,
       "learning_rate": 2.6437126905897967e-07,
-      "loss": 0.1222,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0304,
+      "reward": 0.22916667722165585,
+      "reward_std": 0.29257263243198395,
+      "rewards/accuracy_reward": 0.22916667722165585,
       "step": 747
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 424.8125,
+      "completion_length": 1973.4583740234375,
       "epoch": 0.748,
-      "grad_norm": 4.940523622959794,
-      "kl": 0.3505859375,
+      "grad_norm": 4.645494574688499,
+      "kl": 0.4501953125,
       "learning_rate": 2.631592046130896e-07,
-      "loss": 0.0192,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0097,
+      "reward": 0.3333333544433117,
+      "reward_std": 0.19462472200393677,
+      "rewards/accuracy_reward": 0.3333333544433117,
       "step": 748
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 447.5833435058594,
+      "completion_length": 2048.0,
       "epoch": 0.749,
-      "grad_norm": 14.486858386225327,
-      "kl": 0.75244140625,
+      "grad_norm": 2.9669006587297324,
+      "kl": 0.68359375,
       "learning_rate": 2.6195063523177e-07,
-      "loss": 0.0931,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0274,
+      "reward": 0.1041666679084301,
+      "reward_std": 0.25740431994199753,
+      "rewards/accuracy_reward": 0.1041666679084301,
       "step": 749
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 412.5833435058594,
+      "completion_length": 2012.9166870117188,
       "epoch": 0.75,
-      "grad_norm": 1.4456840039078567,
-      "kl": 0.1439208984375,
+      "grad_norm": 5.542063110331265,
+      "kl": 0.603515625,
       "learning_rate": 2.6074557564105724e-07,
-      "loss": 0.0067,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0125,
+      "reward": 0.1041666679084301,
+      "reward_std": 0.26679350435733795,
+      "rewards/accuracy_reward": 0.1041666679084301,
       "step": 750
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 513.5208435058594,
+      "completion_length": 2048.0,
       "epoch": 0.751,
-      "grad_norm": 35.005057574401924,
-      "kl": 2.38671875,
+      "grad_norm": 3.8450521905389885,
+      "kl": 0.4716796875,
       "learning_rate": 2.595440405242222e-07,
-      "loss": 0.3071,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.07845467701554298,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.0189,
+      "reward": 0.1875000111758709,
+      "reward_std": 0.22604453563690186,
+      "rewards/accuracy_reward": 0.1875000111758709,
       "step": 751
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 446.00001525878906,
+      "completion_length": 1974.7708740234375,
       "epoch": 0.752,
-      "grad_norm": 22.58672685506717,
-      "kl": 0.9561767578125,
+      "grad_norm": 4.22043494149006,
+      "kl": 0.435546875,
       "learning_rate": 2.583460445215911e-07,
-      "loss": 0.1269,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": -0.002,
+      "reward": 0.5625000298023224,
+      "reward_std": 0.32335689663887024,
+      "rewards/accuracy_reward": 0.5625000298023224,
       "step": 752
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 464.62501525878906,
+      "completion_length": 1978.25,
       "epoch": 0.753,
-      "grad_norm": 10.917661375391802,
-      "kl": 0.528076171875,
+      "grad_norm": 4.101479275270948,
+      "kl": 0.73046875,
       "learning_rate": 2.571516022303671e-07,
-      "loss": 0.165,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.0749332457780838,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.0221,
+      "reward": 0.22916667722165585,
+      "reward_std": 0.29257263243198395,
+      "rewards/accuracy_reward": 0.22916667722165585,
       "step": 753
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 547.0208587646484,
+      "completion_length": 1980.4375,
       "epoch": 0.754,
-      "grad_norm": 14.595777318588363,
-      "kl": 2.20703125,
+      "grad_norm": 4.1083877732456315,
+      "kl": 0.53125,
       "learning_rate": 2.5596072820445254e-07,
-      "loss": 0.475,
-      "reward": 0.9583333432674408,
-      "reward_std": 0.1276453733444214,
-      "rewards/tag_count_reward": 0.9583333432674408,
+      "loss": 0.0391,
+      "reward": 0.5416666716337204,
+      "reward_std": 0.3647414296865463,
+      "rewards/accuracy_reward": 0.5416666716337204,
       "step": 754
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 437.7083435058594,
+      "completion_length": 2048.0,
       "epoch": 0.755,
-      "grad_norm": 9.72501428549619,
-      "kl": 0.6739501953125,
+      "grad_norm": 3.36722921774157,
+      "kl": 0.5927734375,
       "learning_rate": 2.547734369542718e-07,
-      "loss": 0.1245,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.0749332457780838,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.0237,
+      "reward": 0.1250000037252903,
+      "reward_std": 0.3295731097459793,
+      "rewards/accuracy_reward": 0.1250000037252903,
       "step": 755
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 441.93751525878906,
+      "completion_length": 2048.0,
       "epoch": 0.756,
-      "grad_norm": 11.940232882356842,
-      "kl": 0.75390625,
+      "grad_norm": 3.9036931017948477,
+      "kl": 0.53125,
       "learning_rate": 2.5358974294659373e-07,
-      "loss": 0.2374,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": 0.0212,
+      "reward": 0.06250000186264515,
+      "reward_std": 0.21650634706020355,
+      "rewards/accuracy_reward": 0.06250000186264515,
       "step": 756
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 425.1458435058594,
+      "completion_length": 2048.0,
       "epoch": 0.757,
-      "grad_norm": 21.25957650725657,
-      "kl": 0.53466796875,
+      "grad_norm": 2.6414759517997872,
+      "kl": 0.84765625,
       "learning_rate": 2.5240966060435674e-07,
-      "loss": 0.0655,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0339,
+      "reward": 0.27083333395421505,
+      "reward_std": 0.26679350435733795,
+      "rewards/accuracy_reward": 0.27083333395421505,
       "step": 757
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 426.2083435058594,
+      "completion_length": 2048.0,
       "epoch": 0.758,
-      "grad_norm": 2.670261773175817,
-      "kl": 0.182861328125,
+      "grad_norm": 3.4475454713907197,
+      "kl": 0.6611328125,
       "learning_rate": 2.512332043064913e-07,
-      "loss": 0.0105,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0264,
+      "reward": 0.3333333432674408,
+      "reward_std": 0.3648904263973236,
+      "rewards/accuracy_reward": 0.3333333432674408,
       "step": 758
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 438.1875,
+      "completion_length": 2048.0,
       "epoch": 0.759,
-      "grad_norm": 3.550420770270506,
-      "kl": 0.375,
+      "grad_norm": 5.47540467600025,
+      "kl": 0.607421875,
       "learning_rate": 2.5006038838774647e-07,
-      "loss": 0.0534,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0243,
+      "reward": 0.1666666716337204,
+      "reward_std": 0.3798602819442749,
+      "rewards/accuracy_reward": 0.1666666716337204,
       "step": 759
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 467.70835876464844,
+      "completion_length": 2048.0,
       "epoch": 0.76,
-      "grad_norm": 9.761605268786091,
-      "kl": 1.1611328125,
+      "grad_norm": 5.129791150904657,
+      "kl": 0.47265625,
       "learning_rate": 2.488912271385139e-07,
-      "loss": 0.2286,
-      "reward": 0.9635416865348816,
-      "reward_std": 0.08307896926999092,
-      "rewards/tag_count_reward": 0.9635416865348816,
+      "loss": 0.0189,
+      "reward": 0.2708333432674408,
+      "reward_std": 0.4370592087507248,
+      "rewards/accuracy_reward": 0.2708333432674408,
       "step": 760
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 574.4791870117188,
+      "completion_length": 2025.1666870117188,
       "epoch": 0.761,
-      "grad_norm": 24.51004058459727,
-      "kl": 1.64453125,
+      "grad_norm": 4.695883641697442,
+      "kl": 0.744140625,
       "learning_rate": 2.4772573480465445e-07,
-      "loss": 0.5824,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.06041248142719269,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0299,
+      "reward": 0.1666666716337204,
+      "reward_std": 0.29821331799030304,
+      "rewards/accuracy_reward": 0.1666666716337204,
       "step": 761
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 485.06251525878906,
+      "completion_length": 2044.75,
       "epoch": 0.762,
-      "grad_norm": 10.02285213561004,
-      "kl": 0.9228515625,
+      "grad_norm": 14.510992684341703,
+      "kl": 0.638671875,
       "learning_rate": 2.465639255873246e-07,
-      "loss": 0.1431,
-      "reward": 0.9895833730697632,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833730697632,
+      "loss": 0.0253,
+      "reward": 0.1875000074505806,
+      "reward_std": 0.3898850083351135,
+      "rewards/accuracy_reward": 0.1875000074505806,
       "step": 762
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 424.25001525878906,
+      "completion_length": 2048.0,
       "epoch": 0.763,
-      "grad_norm": 12.319177988211461,
-      "kl": 0.7608642578125,
+      "grad_norm": 5.034788907282881,
+      "kl": 0.6328125,
       "learning_rate": 2.454058136428027e-07,
-      "loss": 0.0391,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0253,
+      "reward": 0.229166679084301,
+      "reward_std": 0.38796256482601166,
+      "rewards/accuracy_reward": 0.229166679084301,
       "step": 763
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 467.3125,
+      "completion_length": 2048.0,
       "epoch": 0.764,
-      "grad_norm": 16.438634033001705,
-      "kl": 1.4501953125,
+      "grad_norm": 6.697512691114286,
+      "kl": 0.59765625,
       "learning_rate": 2.4425141308231765e-07,
-      "loss": 0.2977,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": 0.0239,
+      "reward": 0.3750000149011612,
+      "reward_std": 0.4665868580341339,
+      "rewards/accuracy_reward": 0.3750000149011612,
       "step": 764
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 447.9375,
+      "completion_length": 2012.7083740234375,
       "epoch": 0.765,
-      "grad_norm": 1.407361201542416,
-      "kl": 0.16259765625,
+      "grad_norm": 6.689706854666545,
+      "kl": 0.4365234375,
       "learning_rate": 2.4310073797187573e-07,
-      "loss": 0.0066,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0176,
+      "reward": 0.5625000149011612,
+      "reward_std": 0.3076914846897125,
+      "rewards/accuracy_reward": 0.5625000149011612,
       "step": 765
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 427.7291717529297,
+      "completion_length": 2048.0,
       "epoch": 0.766,
-      "grad_norm": 4.688693636755467,
-      "kl": 0.31298828125,
+      "grad_norm": 4.490492717099603,
+      "kl": 0.990234375,
       "learning_rate": 2.4195380233209006e-07,
-      "loss": 0.0387,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0397,
+      "reward": 0.2083333358168602,
+      "reward_std": 0.3251829594373703,
+      "rewards/accuracy_reward": 0.2083333358168602,
       "step": 766
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 508.93751525878906,
+      "completion_length": 2048.0,
       "epoch": 0.767,
-      "grad_norm": 5.683735747294983,
-      "kl": 1.4609375,
+      "grad_norm": 6.410986292131669,
+      "kl": 0.859375,
       "learning_rate": 2.408106201380097e-07,
-      "loss": 0.1967,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.06041248142719269,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0344,
+      "reward": 0.4166666716337204,
+      "reward_std": 0.4470839649438858,
+      "rewards/accuracy_reward": 0.4166666716337204,
       "step": 767
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 535.1041717529297,
+      "completion_length": 2048.0,
       "epoch": 0.768,
-      "grad_norm": 9.361688645147792,
-      "kl": 1.2958984375,
+      "grad_norm": 6.520377824589985,
+      "kl": 1.1640625,
       "learning_rate": 2.3967120531894857e-07,
-      "loss": 0.1311,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.058750879019498825,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.0465,
+      "reward": 0.1250000037252903,
+      "reward_std": 0.22040386497974396,
+      "rewards/accuracy_reward": 0.1250000037252903,
       "step": 768
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 520.7916870117188,
+      "completion_length": 2005.9375,
       "epoch": 0.769,
-      "grad_norm": 7.86994387145606,
-      "kl": 0.72802734375,
+      "grad_norm": 8.282421511805477,
+      "kl": 0.921875,
       "learning_rate": 2.38535571758317e-07,
-      "loss": 0.222,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0289,
+      "reward": 0.229166679084301,
+      "reward_std": 0.4213937968015671,
+      "rewards/accuracy_reward": 0.229166679084301,
       "step": 769
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 425.3333435058594,
+      "completion_length": 2031.6875,
       "epoch": 0.77,
-      "grad_norm": 1.8396818228183496,
-      "kl": 0.186767578125,
+      "grad_norm": 5.539312570858117,
+      "kl": 0.55078125,
       "learning_rate": 2.374037332934512e-07,
-      "loss": 0.0035,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0144,
+      "reward": 0.458333358168602,
+      "reward_std": 0.4364236444234848,
+      "rewards/accuracy_reward": 0.458333358168602,
       "step": 770
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 446.5833435058594,
+      "completion_length": 1935.5416870117188,
       "epoch": 0.771,
-      "grad_norm": 16.397728494583784,
-      "kl": 0.681640625,
+      "grad_norm": 3.8006945423875385,
+      "kl": 0.919921875,
       "learning_rate": 2.36275703715446e-07,
-      "loss": 0.3143,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": 0.0278,
+      "reward": 0.2291666679084301,
+      "reward_std": 0.29257264733314514,
+      "rewards/accuracy_reward": 0.2291666679084301,
       "step": 771
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 470.0833435058594,
+      "completion_length": 2008.0208740234375,
       "epoch": 0.772,
-      "grad_norm": 12.503031358735678,
-      "kl": 1.130859375,
+      "grad_norm": 6.478015329975639,
+      "kl": 0.8046875,
       "learning_rate": 2.3515149676898552e-07,
-      "loss": 0.2804,
-      "reward": 0.9687500298023224,
-      "reward_std": 0.08474057167768478,
-      "rewards/tag_count_reward": 0.9687500298023224,
+      "loss": 0.025,
+      "reward": 0.4375000149011612,
+      "reward_std": 0.48980802297592163,
+      "rewards/accuracy_reward": 0.4375000149011612,
       "step": 772
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 462.85418701171875,
+      "completion_length": 2003.0416870117188,
       "epoch": 0.773,
-      "grad_norm": 3.9169964655197944,
-      "kl": 0.38720703125,
+      "grad_norm": 4.199355456595402,
+      "kl": 0.541015625,
       "learning_rate": 2.3403112615217693e-07,
-      "loss": 0.0518,
-      "reward": 0.984375,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0201,
+      "reward": 0.0833333358168602,
+      "reward_std": 0.18523554503917694,
+      "rewards/accuracy_reward": 0.0833333358168602,
       "step": 773
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 416.9166717529297,
+      "completion_length": 1978.8125,
       "epoch": 0.774,
-      "grad_norm": 0.9880321965045378,
-      "kl": 0.142822265625,
+      "grad_norm": 5.9978794540143605,
+      "kl": 0.689453125,
       "learning_rate": 2.3291460551638237e-07,
-      "loss": 0.006,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.058,
+      "reward": 0.6041666865348816,
+      "reward_std": 0.4156641662120819,
+      "rewards/accuracy_reward": 0.6041666865348816,
       "step": 774
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 540.6666870117188,
+      "completion_length": 1981.0625,
       "epoch": 0.775,
-      "grad_norm": 11.726121649739454,
-      "kl": 0.869140625,
+      "grad_norm": 8.109644124117857,
+      "kl": 0.66015625,
       "learning_rate": 2.3180194846605364e-07,
-      "loss": 0.3413,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.07216878235340118,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0571,
+      "reward": 0.4791666716337204,
+      "reward_std": 0.4156641364097595,
+      "rewards/accuracy_reward": 0.4791666716337204,
       "step": 775
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 471.64585876464844,
+      "completion_length": 1973.5625,
       "epoch": 0.776,
-      "grad_norm": 6.582729677504788,
-      "kl": 0.580078125,
+      "grad_norm": 6.53301713075602,
+      "kl": 0.650390625,
       "learning_rate": 2.306931685585657e-07,
-      "loss": 0.0981,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0118,
+      "reward": 0.3958333432674408,
+      "reward_std": 0.4307829737663269,
+      "rewards/accuracy_reward": 0.3958333432674408,
       "step": 776
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 454.3541717529297,
+      "completion_length": 2042.7916870117188,
       "epoch": 0.777,
-      "grad_norm": 30.100361780455177,
-      "kl": 1.84375,
+      "grad_norm": 7.2942080579548145,
+      "kl": 0.51171875,
       "learning_rate": 2.2958827930405162e-07,
-      "loss": 0.3048,
-      "reward": 0.9635416865348816,
-      "reward_std": 0.07955753058195114,
-      "rewards/tag_count_reward": 0.9635416865348816,
+      "loss": 0.0189,
+      "reward": 0.7083333432674408,
+      "reward_std": 0.3314591944217682,
+      "rewards/accuracy_reward": 0.7083333432674408,
       "step": 777
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 475.72918701171875,
+      "completion_length": 2008.2083740234375,
       "epoch": 0.778,
-      "grad_norm": 6.574018009507352,
-      "kl": 0.27490234375,
+      "grad_norm": 8.767488772659986,
+      "kl": 1.041015625,
       "learning_rate": 2.2848729416523859e-07,
-      "loss": 0.0354,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.07216878235340118,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.01,
+      "reward": 0.4166666716337204,
+      "reward_std": 0.487981915473938,
+      "rewards/accuracy_reward": 0.4166666716337204,
       "step": 778
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 408.0,
+      "completion_length": 2010.3333740234375,
       "epoch": 0.779,
-      "grad_norm": 0.47326634644492166,
-      "kl": 0.0831298828125,
+      "grad_norm": 11.497544238969583,
+      "kl": 0.8212890625,
       "learning_rate": 2.2739022655728277e-07,
-      "loss": 0.0035,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0279,
+      "reward": 0.5416666865348816,
+      "reward_std": 0.41502857208251953,
+      "rewards/accuracy_reward": 0.5416666865348816,
       "step": 779
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 520.0208587646484,
+      "completion_length": 2034.9375,
       "epoch": 0.78,
-      "grad_norm": 2.9849829875627867,
-      "kl": 0.51708984375,
+      "grad_norm": 5.875898337677492,
+      "kl": 0.947265625,
       "learning_rate": 2.2629708984760706e-07,
-      "loss": 0.0061,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0332,
+      "reward": 0.2708333358168602,
+      "reward_std": 0.3898850083351135,
+      "rewards/accuracy_reward": 0.2708333358168602,
       "step": 780
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 465.1458435058594,
+      "completion_length": 1936.0,
       "epoch": 0.781,
-      "grad_norm": 0.4898002516853844,
-      "kl": 0.077392578125,
+      "grad_norm": 7.690841376974938,
+      "kl": 1.3359375,
       "learning_rate": 2.2520789735573704e-07,
-      "loss": 0.0035,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.026,
+      "reward": 0.3125000149011612,
+      "reward_std": 0.4307829737663269,
+      "rewards/accuracy_reward": 0.3125000149011612,
       "step": 781
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 426.1666717529297,
+      "completion_length": 1986.8125,
       "epoch": 0.782,
-      "grad_norm": 2.203243601671771,
-      "kl": 0.195556640625,
+      "grad_norm": 6.091309613017156,
+      "kl": 1.111328125,
       "learning_rate": 2.2412266235313973e-07,
-      "loss": 0.0108,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0504,
+      "reward": 0.4791666679084301,
+      "reward_std": 0.31381870806217194,
+      "rewards/accuracy_reward": 0.4791666679084301,
       "step": 782
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 412.9583435058594,
+      "completion_length": 1970.5625610351562,
       "epoch": 0.783,
-      "grad_norm": 2.4466254859091947,
-      "kl": 0.171875,
+      "grad_norm": 13.172797614712776,
+      "kl": 1.1328125,
       "learning_rate": 2.230413980630609e-07,
-      "loss": 0.0147,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0557,
+      "reward": 0.3333333432674408,
+      "reward_std": 0.3861365020275116,
+      "rewards/accuracy_reward": 0.3333333432674408,
       "step": 783
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 439.8958435058594,
+      "completion_length": 1941.7083740234375,
       "epoch": 0.784,
-      "grad_norm": 4.002690913646053,
-      "kl": 0.295654296875,
+      "grad_norm": 13.841528815125763,
+      "kl": 1.1640625,
       "learning_rate": 2.2196411766036487e-07,
-      "loss": 0.0801,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0486,
+      "reward": 0.3125000074505806,
+      "reward_std": 0.3334706276655197,
+      "rewards/accuracy_reward": 0.3125000074505806,
       "step": 784
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 448.16668701171875,
+      "completion_length": 2029.1458740234375,
       "epoch": 0.785,
-      "grad_norm": 18.272707650400818,
-      "kl": 0.798828125,
+      "grad_norm": 11.04183103746986,
+      "kl": 1.107421875,
       "learning_rate": 2.2089083427137329e-07,
-      "loss": 0.216,
-      "reward": 0.9687500298023224,
-      "reward_std": 0.10825317353010178,
-      "rewards/tag_count_reward": 0.9687500298023224,
+      "loss": 0.0322,
+      "reward": 0.3750000111758709,
+      "reward_std": 0.3905205726623535,
+      "rewards/accuracy_reward": 0.3750000111758709,
       "step": 785
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 504.87501525878906,
+      "completion_length": 1979.0416870117188,
       "epoch": 0.786,
-      "grad_norm": 4.5120995319694845,
-      "kl": 0.80712890625,
+      "grad_norm": 5.3634703171167075,
+      "kl": 0.974609375,
       "learning_rate": 2.1982156097370557e-07,
-      "loss": 0.1073,
-      "reward": 0.984375,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0398,
+      "reward": 0.2083333432674408,
+      "reward_std": 0.22040385007858276,
+      "rewards/accuracy_reward": 0.2083333432674408,
       "step": 786
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 470.3333435058594,
+      "completion_length": 2041.9375610351562,
       "epoch": 0.787,
-      "grad_norm": 8.290006005097963,
-      "kl": 0.7265625,
+      "grad_norm": 6.6116417789943345,
+      "kl": 1.0,
       "learning_rate": 2.1875631079611956e-07,
-      "loss": 0.0023,
-      "reward": 0.9739583730697632,
-      "reward_std": 0.09021097794175148,
-      "rewards/tag_count_reward": 0.9739583730697632,
+      "loss": 0.0406,
+      "reward": 0.625,
+      "reward_std": 0.22613351047039032,
+      "rewards/accuracy_reward": 0.625,
       "step": 787
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 551.8333740234375,
+      "completion_length": 2011.75,
       "epoch": 0.788,
-      "grad_norm": 6.163806278693715,
-      "kl": 0.77197265625,
+      "grad_norm": 13.428863554034907,
+      "kl": 1.4453125,
       "learning_rate": 2.1769509671835223e-07,
-      "loss": 0.1466,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.04865618050098419,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0589,
+      "reward": 0.1875,
+      "reward_std": 0.3139677122235298,
+      "rewards/accuracy_reward": 0.1875,
       "step": 788
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 489.4375,
+      "completion_length": 2013.5625,
       "epoch": 0.789,
-      "grad_norm": 15.557960983334132,
-      "kl": 1.03857421875,
+      "grad_norm": 12.635608148926401,
+      "kl": 0.818359375,
       "learning_rate": 2.166379316709625e-07,
-      "loss": 0.3285,
-      "reward": 0.9895833730697632,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833730697632,
+      "loss": 0.0254,
+      "reward": 0.16666667722165585,
+      "reward_std": 0.2748958021402359,
+      "rewards/accuracy_reward": 0.16666667722165585,
       "step": 789
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 477.27085876464844,
+      "completion_length": 1910.8333740234375,
       "epoch": 0.79,
-      "grad_norm": 10.413647380902116,
-      "kl": 0.8818359375,
+      "grad_norm": 13.196482937387428,
+      "kl": 1.103515625,
       "learning_rate": 2.1558482853517253e-07,
-      "loss": 0.2329,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0165,
+      "reward": 0.0833333358168602,
+      "reward_std": 0.28867512941360474,
+      "rewards/accuracy_reward": 0.0833333358168602,
       "step": 790
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 401.9166717529297,
+      "completion_length": 2048.0,
       "epoch": 0.791,
-      "grad_norm": 3.8936337344497334,
-      "kl": 0.490234375,
+      "grad_norm": 5.228179838324819,
+      "kl": 0.662109375,
       "learning_rate": 2.1453580014271203e-07,
-      "loss": 0.0582,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0265,
+      "reward": 0.5000000149011612,
+      "reward_std": 0.42877157032489777,
+      "rewards/accuracy_reward": 0.5000000149011612,
       "step": 791
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 406.8333435058594,
+      "completion_length": 2013.7291870117188,
       "epoch": 0.792,
-      "grad_norm": 3.8365345601196426,
-      "kl": 0.2802734375,
+      "grad_norm": 7.463446585550285,
+      "kl": 0.8515625,
       "learning_rate": 2.134908592756607e-07,
-      "loss": 0.0132,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0358,
+      "reward": 0.2083333432674408,
+      "reward_std": 0.29821331799030304,
+      "rewards/accuracy_reward": 0.2083333432674408,
       "step": 792
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 454.0,
+      "completion_length": 2012.7916870117188,
       "epoch": 0.793,
-      "grad_norm": 4.896164030024204,
-      "kl": 0.490234375,
+      "grad_norm": 6.474989668531329,
+      "kl": 1.3359375,
       "learning_rate": 2.124500186662932e-07,
-      "loss": 0.068,
-      "reward": 0.984375,
-      "reward_std": 0.03884884715080261,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0543,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 793
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 448.1666717529297,
+      "completion_length": 2048.0,
       "epoch": 0.794,
-      "grad_norm": 7.086922184767771,
-      "kl": 0.6494140625,
+      "grad_norm": 4.1689587016197285,
+      "kl": 0.79296875,
       "learning_rate": 2.1141329099692406e-07,
-      "loss": 0.0308,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0318,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.14433756470680237,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 794
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 475.375,
+      "completion_length": 2048.0,
       "epoch": 0.795,
-      "grad_norm": 1.7909734564065407,
-      "kl": 0.15576171875,
+      "grad_norm": 2.948724898084049,
+      "kl": 0.841796875,
       "learning_rate": 2.1038068889975259e-07,
-      "loss": 0.0317,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0337,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.09731236100196838,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 795
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 497.0833435058594,
+      "completion_length": 1973.0208740234375,
       "epoch": 0.796,
-      "grad_norm": 12.102755712800246,
-      "kl": 0.6318359375,
+      "grad_norm": 7.806611762478713,
+      "kl": 0.83984375,
       "learning_rate": 2.0935222495670968e-07,
-      "loss": 0.1842,
-      "reward": 0.9895833730697632,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833730697632,
+      "loss": 0.0172,
+      "reward": 0.1250000037252903,
+      "reward_std": 0.2825479209423065,
+      "rewards/accuracy_reward": 0.1250000037252903,
       "step": 796
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 573.7916870117188,
+      "completion_length": 2007.7708740234375,
       "epoch": 0.797,
-      "grad_norm": 12.78765245301671,
-      "kl": 0.822265625,
+      "grad_norm": 4.096080530496686,
+      "kl": 1.07421875,
       "learning_rate": 2.0832791169930363e-07,
-      "loss": 0.1833,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": 0.0435,
+      "reward": 0.08333333395421505,
+      "reward_std": 0.18523553758859634,
+      "rewards/accuracy_reward": 0.08333333395421505,
       "step": 797
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 478.3541717529297,
+      "completion_length": 2011.2083740234375,
       "epoch": 0.798,
-      "grad_norm": 1.9918339123953115,
-      "kl": 0.31591796875,
+      "grad_norm": 5.208473539787504,
+      "kl": 0.865234375,
       "learning_rate": 2.0730776160846853e-07,
-      "loss": 0.0254,
-      "reward": 0.984375,
-      "reward_std": 0.03884884715080261,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0197,
+      "reward": 0.3333333432674408,
+      "reward_std": 0.46966953575611115,
+      "rewards/accuracy_reward": 0.3333333432674408,
       "step": 798
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 533.3958587646484,
+      "completion_length": 2009.875,
       "epoch": 0.799,
-      "grad_norm": 12.82764672497113,
-      "kl": 0.98779296875,
+      "grad_norm": 2.6796559708162233,
+      "kl": 0.888671875,
       "learning_rate": 2.0629178711441115e-07,
-      "loss": 0.2835,
-      "reward": 0.9687500298023224,
-      "reward_std": 0.10825317353010178,
-      "rewards/tag_count_reward": 0.9687500298023224,
+      "loss": 0.0268,
+      "reward": 0.14583333395421505,
+      "reward_std": 0.20272701978683472,
+      "rewards/accuracy_reward": 0.14583333395421505,
       "step": 799
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 481.3958435058594,
+      "completion_length": 1857.1666870117188,
       "epoch": 0.8,
-      "grad_norm": 3.39840233989013,
-      "kl": 0.1318359375,
+      "grad_norm": 2.919529471685376,
+      "kl": 0.7578125,
       "learning_rate": 2.0528000059645995e-07,
-      "loss": 0.0493,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0315,
+      "reward": 0.1458333395421505,
+      "reward_std": 0.3395978510379791,
+      "rewards/accuracy_reward": 0.1458333395421505,
       "step": 800
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 440.2916717529297,
+      "completion_length": 1899.7083740234375,
       "epoch": 0.801,
-      "grad_norm": 1.17039551897165,
-      "kl": 0.14599609375,
+      "grad_norm": 4.745493104023661,
+      "kl": 0.7265625,
       "learning_rate": 2.042724143829146e-07,
-      "loss": 0.0057,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0749,
+      "reward": 0.2708333432674408,
+      "reward_std": 0.2983023002743721,
+      "rewards/accuracy_reward": 0.2708333432674408,
       "step": 801
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 441.60418701171875,
+      "completion_length": 1940.7083740234375,
       "epoch": 0.802,
-      "grad_norm": 0.6409417964283548,
-      "kl": 0.098388671875,
+      "grad_norm": 4.2364565490962685,
+      "kl": 0.728515625,
       "learning_rate": 2.032690407508949e-07,
-      "loss": 0.0039,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.1031,
+      "reward": 0.4166666716337204,
+      "reward_std": 0.35477669537067413,
+      "rewards/accuracy_reward": 0.4166666716337204,
       "step": 802
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 438.3125,
+      "completion_length": 1934.4375610351562,
       "epoch": 0.803,
-      "grad_norm": 0.5923060952205192,
-      "kl": 0.084716796875,
+      "grad_norm": 2.3022405723869626,
+      "kl": 0.783203125,
       "learning_rate": 2.0226989192619204e-07,
-      "loss": 0.0035,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0326,
+      "reward": 0.0833333358168602,
+      "reward_std": 0.12309150397777557,
+      "rewards/accuracy_reward": 0.0833333358168602,
       "step": 803
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 443.06251525878906,
+      "completion_length": 1974.9166870117188,
       "epoch": 0.804,
-      "grad_norm": 1.848745051481109,
-      "kl": 0.136962890625,
+      "grad_norm": 4.456755829849114,
+      "kl": 0.666015625,
       "learning_rate": 2.0127498008311922e-07,
-      "loss": 0.0323,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.031,
+      "reward": 0.2916666865348816,
+      "reward_std": 0.35477669537067413,
+      "rewards/accuracy_reward": 0.2916666865348816,
       "step": 804
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 420.9166717529297,
+      "completion_length": 1861.75,
       "epoch": 0.805,
-      "grad_norm": 4.013705556924666,
-      "kl": 0.34765625,
+      "grad_norm": 3.359246115258536,
+      "kl": 0.66796875,
       "learning_rate": 2.0028431734436308e-07,
-      "loss": 0.0543,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0219,
+      "reward": 0.2916666716337204,
+      "reward_std": 0.2536497414112091,
+      "rewards/accuracy_reward": 0.2916666716337204,
       "step": 805
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 483.97918701171875,
+      "completion_length": 2048.0,
       "epoch": 0.806,
-      "grad_norm": 3.2232372608499174,
-      "kl": 0.3173828125,
+      "grad_norm": 3.5622587350257002,
+      "kl": 0.66796875,
       "learning_rate": 1.9929791578083655e-07,
-      "loss": 0.0114,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0267,
+      "reward": 0.4583333432674408,
+      "reward_std": 0.42877158522605896,
+      "rewards/accuracy_reward": 0.4583333432674408,
       "step": 806
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 498.47918701171875,
+      "completion_length": 1907.0833740234375,
       "epoch": 0.807,
-      "grad_norm": 5.401087737272918,
-      "kl": 0.71484375,
+      "grad_norm": 4.644311696173289,
+      "kl": 0.822265625,
       "learning_rate": 1.9831578741153155e-07,
-      "loss": 0.1851,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0982,
+      "reward": 0.20833333395421505,
+      "reward_std": 0.26742906868457794,
+      "rewards/accuracy_reward": 0.20833333395421505,
       "step": 807
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 440.8333435058594,
+      "completion_length": 1975.3958740234375,
       "epoch": 0.808,
-      "grad_norm": 2.0006096032089475,
-      "kl": 0.148193359375,
+      "grad_norm": 3.044221268509813,
+      "kl": 0.87109375,
       "learning_rate": 1.9733794420337213e-07,
-      "loss": 0.0078,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0244,
+      "reward": 0.25000000558793545,
+      "reward_std": 0.331459179520607,
+      "rewards/accuracy_reward": 0.25000000558793545,
       "step": 808
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 394.18751525878906,
+      "completion_length": 1976.041748046875,
       "epoch": 0.809,
-      "grad_norm": 0.5031508156464227,
-      "kl": 0.05029296875,
+      "grad_norm": 2.681788368408633,
+      "kl": 1.048828125,
       "learning_rate": 1.9636439807106912e-07,
-      "loss": 0.0021,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0589,
+      "reward": 0.18750000558793545,
+      "reward_std": 0.19526028633117676,
+      "rewards/accuracy_reward": 0.18750000558793545,
       "step": 809
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 441.04168701171875,
+      "completion_length": 2048.0,
       "epoch": 0.81,
-      "grad_norm": 0.6605051513844795,
-      "kl": 0.08056640625,
+      "grad_norm": 3.294777435481099,
+      "kl": 0.984375,
       "learning_rate": 1.9539516087697517e-07,
-      "loss": 0.0033,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0394,
+      "reward": 0.1875,
+      "reward_std": 0.11306675523519516,
+      "rewards/accuracy_reward": 0.1875,
       "step": 810
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 422.0625,
+      "completion_length": 2048.0,
       "epoch": 0.811,
-      "grad_norm": 3.881878608245351,
-      "kl": 0.23193359375,
+      "grad_norm": 2.8829035305286506,
+      "kl": 0.640625,
       "learning_rate": 1.944302444309393e-07,
-      "loss": 0.0133,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0256,
+      "reward": 0.2083333395421505,
+      "reward_std": 0.3251829743385315,
+      "rewards/accuracy_reward": 0.2083333395421505,
       "step": 811
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 349.4166717529297,
+      "completion_length": 2012.2916870117188,
       "epoch": 0.812,
-      "grad_norm": 3.3941575923116543,
-      "kl": 0.067138671875,
+      "grad_norm": 2.8508710256962795,
+      "kl": 0.650390625,
       "learning_rate": 1.934696604901642e-07,
-      "loss": 0.0356,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0305,
+      "reward": 0.2708333395421505,
+      "reward_std": 0.3076914846897125,
+      "rewards/accuracy_reward": 0.2708333395421505,
       "step": 812
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 391.625,
+      "completion_length": 2041.9791870117188,
       "epoch": 0.813,
-      "grad_norm": 2.354593368951386,
-      "kl": 0.1708984375,
+      "grad_norm": 4.130022346163883,
+      "kl": 0.71875,
       "learning_rate": 1.9251342075906179e-07,
-      "loss": 0.0081,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0289,
+      "reward": 0.4583333432674408,
+      "reward_std": 0.29193708300590515,
+      "rewards/accuracy_reward": 0.4583333432674408,
       "step": 813
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 508.62501525878906,
+      "completion_length": 2013.3541870117188,
       "epoch": 0.814,
-      "grad_norm": 4.684002035921945,
-      "kl": 0.2940673828125,
+      "grad_norm": 4.062900067642915,
+      "kl": 0.7265625,
       "learning_rate": 1.915615368891117e-07,
-      "loss": 0.0818,
-      "reward": 0.9635416865348816,
-      "reward_std": 0.07457557320594788,
-      "rewards/tag_count_reward": 0.9635416865348816,
+      "loss": 0.0372,
+      "reward": 0.4375000149011612,
+      "reward_std": 0.2574043273925781,
+      "rewards/accuracy_reward": 0.4375000149011612,
       "step": 814
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 388.2083435058594,
+      "completion_length": 2013.8541870117188,
       "epoch": 0.815,
-      "grad_norm": 7.01050645008269,
-      "kl": 0.387939453125,
+      "grad_norm": 2.9198649743658334,
+      "kl": 0.703125,
       "learning_rate": 1.9061402047871833e-07,
-      "loss": 0.0556,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0286,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.09731236100196838,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 815
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 384.7708435058594,
+      "completion_length": 2048.0,
       "epoch": 0.816,
-      "grad_norm": 2.2092743578327623,
-      "kl": 0.06591796875,
+      "grad_norm": 5.463395312884968,
+      "kl": 0.796875,
       "learning_rate": 1.8967088307307e-07,
-      "loss": 0.0047,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0319,
+      "reward": 0.06250000186264515,
+      "reward_std": 0.21650634706020355,
+      "rewards/accuracy_reward": 0.06250000186264515,
       "step": 816
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 500.3125305175781,
+      "completion_length": 1970.5833740234375,
       "epoch": 0.817,
-      "grad_norm": 0.5326382898866008,
-      "kl": 0.0927734375,
+      "grad_norm": 2.391740013991015,
+      "kl": 0.533203125,
       "learning_rate": 1.887321361639985e-07,
-      "loss": 0.0037,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0098,
+      "reward": 0.5208333432674408,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.5208333432674408,
       "step": 817
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 463.35418701171875,
+      "completion_length": 2048.0,
       "epoch": 0.818,
-      "grad_norm": 23.222843148752947,
-      "kl": 1.390625,
+      "grad_norm": 6.690785312765727,
+      "kl": 0.8359375,
       "learning_rate": 1.8779779118983867e-07,
-      "loss": 0.3004,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0334,
+      "reward": 0.4375,
+      "reward_std": 0.3703821152448654,
+      "rewards/accuracy_reward": 0.4375,
       "step": 818
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 454.2500305175781,
+      "completion_length": 2048.0,
       "epoch": 0.819,
-      "grad_norm": 2.7247414932960843,
-      "kl": 0.179443359375,
+      "grad_norm": 5.100082071897895,
+      "kl": 0.517578125,
       "learning_rate": 1.8686785953528922e-07,
-      "loss": 0.008,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0207,
+      "reward": 0.29166667722165585,
+      "reward_std": 0.14433756470680237,
+      "rewards/accuracy_reward": 0.29166667722165585,
       "step": 819
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 405.9583435058594,
+      "completion_length": 2048.0,
       "epoch": 0.82,
-      "grad_norm": 2.4711903035532443,
-      "kl": 0.107177734375,
+      "grad_norm": 7.817353534275112,
+      "kl": 0.4287109375,
       "learning_rate": 1.8594235253127372e-07,
-      "loss": 0.0236,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0171,
+      "reward": 0.5416666865348816,
+      "reward_std": 0.24164992570877075,
+      "rewards/accuracy_reward": 0.5416666865348816,
       "step": 820
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 514.6875,
+      "completion_length": 2048.0,
       "epoch": 0.821,
-      "grad_norm": 6.694991948872537,
-      "kl": 0.3466796875,
+      "grad_norm": 7.419134085553707,
+      "kl": 0.50390625,
       "learning_rate": 1.850212814548031e-07,
-      "loss": 0.1343,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": 0.0202,
+      "reward": 0.5000000111758709,
+      "reward_std": 0.24164992570877075,
+      "rewards/accuracy_reward": 0.5000000111758709,
       "step": 821
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 404.37501525878906,
+      "completion_length": 2048.0,
       "epoch": 0.822,
-      "grad_norm": 0.40739460926817,
-      "kl": 0.0841064453125,
+      "grad_norm": 6.520455765192643,
+      "kl": 0.48828125,
       "learning_rate": 1.8410465752883758e-07,
-      "loss": 0.0034,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0196,
+      "reward": 0.2708333395421505,
+      "reward_std": 0.16948114335536957,
+      "rewards/accuracy_reward": 0.2708333395421505,
       "step": 822
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 476.64585876464844,
+      "completion_length": 2040.4166870117188,
       "epoch": 0.823,
-      "grad_norm": 11.307479964352174,
-      "kl": 0.95556640625,
+      "grad_norm": 7.0856149220560996,
+      "kl": 0.6044921875,
       "learning_rate": 1.8319249192215055e-07,
-      "loss": 0.1691,
-      "reward": 0.984375,
-      "reward_std": 0.03884884715080261,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0238,
+      "reward": 0.29166667722165585,
+      "reward_std": 0.3083270341157913,
+      "rewards/accuracy_reward": 0.29166667722165585,
       "step": 823
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 405.79168701171875,
+      "completion_length": 2048.0,
       "epoch": 0.824,
-      "grad_norm": 0.8410365687756945,
-      "kl": 0.095458984375,
+      "grad_norm": 13.130459136751622,
+      "kl": 0.5224609375,
       "learning_rate": 1.822847957491922e-07,
-      "loss": 0.0045,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0209,
+      "reward": 0.3333333432674408,
+      "reward_std": 0.33896230161190033,
+      "rewards/accuracy_reward": 0.3333333432674408,
       "step": 824
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 452.5416717529297,
+      "completion_length": 2048.0,
       "epoch": 0.825,
-      "grad_norm": 5.302986400500268,
-      "kl": 0.41162109375,
+      "grad_norm": 5.970521040519888,
+      "kl": 0.611328125,
       "learning_rate": 1.8138158006995363e-07,
-      "loss": 0.164,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": 0.0244,
+      "reward": 0.06250000186264515,
+      "reward_std": 0.16948114335536957,
+      "rewards/accuracy_reward": 0.06250000186264515,
       "step": 825
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 511.66668701171875,
+      "completion_length": 2048.0,
       "epoch": 0.826,
-      "grad_norm": 21.20486140542435,
-      "kl": 2.265625,
+      "grad_norm": 9.042210468081604,
+      "kl": 0.447265625,
       "learning_rate": 1.804828558898332e-07,
-      "loss": 0.3956,
-      "reward": 0.9427083432674408,
-      "reward_std": 0.1291983723640442,
-      "rewards/tag_count_reward": 0.9427083432674408,
+      "loss": 0.0179,
+      "reward": 0.4166666679084301,
+      "reward_std": 0.36474141478538513,
+      "rewards/accuracy_reward": 0.4166666679084301,
       "step": 826
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 439.2708435058594,
+      "completion_length": 2048.0,
       "epoch": 0.827,
-      "grad_norm": 0.33843770005717105,
-      "kl": 0.0599365234375,
+      "grad_norm": 7.6128853602444,
+      "kl": 0.658203125,
       "learning_rate": 1.7958863415950112e-07,
-      "loss": 0.0025,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0264,
+      "reward": 0.4583333432674408,
+      "reward_std": 0.29193708300590515,
+      "rewards/accuracy_reward": 0.4583333432674408,
       "step": 827
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 513.2500152587891,
+      "completion_length": 2018.875,
       "epoch": 0.828,
-      "grad_norm": 3.3190965305319784,
-      "kl": 0.54541015625,
+      "grad_norm": 13.208156953463305,
+      "kl": 0.5625,
       "learning_rate": 1.7869892577476722e-07,
-      "loss": 0.0249,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0156,
+      "reward": 0.2916666716337204,
+      "reward_std": 0.4287715405225754,
+      "rewards/accuracy_reward": 0.2916666716337204,
       "step": 828
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 455.6875,
+      "completion_length": 2043.375,
       "epoch": 0.829,
-      "grad_norm": 2.3840650360632987,
-      "kl": 0.14794921875,
+      "grad_norm": 7.232395546337291,
+      "kl": 0.677734375,
       "learning_rate": 1.7781374157644713e-07,
-      "loss": 0.0412,
-      "reward": 0.984375,
-      "reward_std": 0.03884884715080261,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.024,
+      "reward": 0.22916667722165585,
+      "reward_std": 0.2574043273925781,
+      "rewards/accuracy_reward": 0.22916667722165585,
       "step": 829
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 401.0208435058594,
+      "completion_length": 2048.0,
       "epoch": 0.83,
-      "grad_norm": 2.090446063492981,
-      "kl": 0.228515625,
+      "grad_norm": 11.714755128459569,
+      "kl": 0.6484375,
       "learning_rate": 1.7693309235023127e-07,
-      "loss": 0.035,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0259,
+      "reward": 0.4791666865348816,
+      "reward_std": 0.3547166883945465,
+      "rewards/accuracy_reward": 0.4791666865348816,
       "step": 830
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 486.6458435058594,
+      "completion_length": 2042.6041870117188,
       "epoch": 0.831,
-      "grad_norm": 3.0077010271823728,
-      "kl": 0.3515625,
+      "grad_norm": 9.586085522442374,
+      "kl": 0.5625,
       "learning_rate": 1.7605698882655233e-07,
-      "loss": 0.0079,
-      "reward": 0.984375,
-      "reward_std": 0.03884884715080261,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0206,
+      "reward": 0.2708333432674408,
+      "reward_std": 0.3547166883945465,
+      "rewards/accuracy_reward": 0.2708333432674408,
       "step": 831
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 356.75001525878906,
+      "completion_length": 2048.0,
       "epoch": 0.832,
-      "grad_norm": 4.850165404763736,
-      "kl": 0.256591796875,
+      "grad_norm": 12.127500419558599,
+      "kl": 0.828125,
       "learning_rate": 1.7518544168045524e-07,
-      "loss": 0.0276,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0332,
+      "reward": 0.1041666679084301,
+      "reward_std": 0.26679350435733795,
+      "rewards/accuracy_reward": 0.1041666679084301,
       "step": 832
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 447.54168701171875,
+      "completion_length": 2048.0,
       "epoch": 0.833,
-      "grad_norm": 0.7953158234930094,
-      "kl": 0.11376953125,
+      "grad_norm": 7.62530675703308,
+      "kl": 0.5283203125,
       "learning_rate": 1.743184615314671e-07,
-      "loss": 0.0045,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0211,
+      "reward": 0.4583333432674408,
+      "reward_std": 0.29193709790706635,
+      "rewards/accuracy_reward": 0.4583333432674408,
       "step": 833
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 434.4583435058594,
+      "completion_length": 2048.0,
       "epoch": 0.834,
-      "grad_norm": 8.22832980491481,
-      "kl": 0.437744140625,
+      "grad_norm": 15.664439814327075,
+      "kl": 0.78125,
       "learning_rate": 1.7345605894346726e-07,
-      "loss": 0.1007,
-      "reward": 0.9895833730697632,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833730697632,
+      "loss": 0.0313,
+      "reward": 0.2083333432674408,
+      "reward_std": 0.4056393951177597,
+      "rewards/accuracy_reward": 0.2083333432674408,
       "step": 834
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 465.3958435058594,
+      "completion_length": 2048.0,
       "epoch": 0.835,
-      "grad_norm": 2.144733225418256,
-      "kl": 0.396240234375,
+      "grad_norm": 5.887703909796236,
+      "kl": 0.921875,
       "learning_rate": 1.7259824442455923e-07,
-      "loss": 0.0258,
-      "reward": 0.96875,
-      "reward_std": 0.04985775053501129,
-      "rewards/tag_count_reward": 0.96875,
+      "loss": 0.0368,
+      "reward": 0.2708333432674408,
+      "reward_std": 0.16948114335536957,
+      "rewards/accuracy_reward": 0.2708333432674408,
       "step": 835
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 363.1666717529297,
+      "completion_length": 2048.0,
       "epoch": 0.836,
-      "grad_norm": 0.14361266387976063,
-      "kl": 0.044677734375,
+      "grad_norm": 15.15758946818134,
+      "kl": 0.892578125,
       "learning_rate": 1.7174502842694212e-07,
-      "loss": 0.0019,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0358,
+      "reward": 0.3750000149011612,
+      "reward_std": 0.4207582473754883,
+      "rewards/accuracy_reward": 0.3750000149011612,
       "step": 836
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 519.4166870117188,
+      "completion_length": 2048.0,
       "epoch": 0.837,
-      "grad_norm": 2.665693523346552,
-      "kl": 0.15869140625,
+      "grad_norm": 8.803974852179108,
+      "kl": 0.97265625,
       "learning_rate": 1.7089642134678364e-07,
-      "loss": -0.015,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": 0.0389,
+      "reward": 0.3541666679084301,
+      "reward_std": 0.3961612284183502,
+      "rewards/accuracy_reward": 0.3541666679084301,
       "step": 837
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 448.4791717529297,
+      "completion_length": 2042.625,
       "epoch": 0.838,
-      "grad_norm": 1.2240220774373272,
-      "kl": 0.126220703125,
+      "grad_norm": 14.169356134967185,
+      "kl": 0.95703125,
       "learning_rate": 1.7005243352409333e-07,
-      "loss": 0.0057,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0359,
+      "reward": 0.3333333432674408,
+      "reward_std": 0.20090095698833466,
+      "rewards/accuracy_reward": 0.3333333432674408,
       "step": 838
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 452.6875,
+      "completion_length": 2048.0,
       "epoch": 0.839,
-      "grad_norm": 1.5444835787344973,
-      "kl": 0.1484375,
+      "grad_norm": 15.472068982902956,
+      "kl": 1.228515625,
       "learning_rate": 1.6921307524259625e-07,
-      "loss": 0.0167,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0491,
+      "reward": 0.3541666716337204,
+      "reward_std": 0.3409373611211777,
+      "rewards/accuracy_reward": 0.3541666716337204,
       "step": 839
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 472.7083435058594,
+      "completion_length": 2034.5416870117188,
       "epoch": 0.84,
-      "grad_norm": 2.461122633133082,
-      "kl": 0.33251953125,
+      "grad_norm": 9.617647736633751,
+      "kl": 1.2734375,
       "learning_rate": 1.6837835672960831e-07,
-      "loss": 0.0258,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": 0.0486,
+      "reward": 0.2291666679084301,
+      "reward_std": 0.32335688173770905,
+      "rewards/accuracy_reward": 0.2291666679084301,
       "step": 840
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 459.8333435058594,
+      "completion_length": 2048.0,
       "epoch": 0.841,
-      "grad_norm": 4.059867011479376,
-      "kl": 0.373779296875,
+      "grad_norm": 7.176078576239386,
+      "kl": 0.88671875,
       "learning_rate": 1.6754828815591131e-07,
-      "loss": 0.1308,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0355,
+      "reward": 0.4791666865348816,
+      "reward_std": 0.38988499343395233,
+      "rewards/accuracy_reward": 0.4791666865348816,
       "step": 841
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 466.0833435058594,
+      "completion_length": 2021.6875,
       "epoch": 0.842,
-      "grad_norm": 13.052029701951339,
-      "kl": 0.919921875,
+      "grad_norm": 11.740312184460981,
+      "kl": 1.13671875,
       "learning_rate": 1.6672287963562852e-07,
-      "loss": 0.2405,
-      "reward": 0.9635416865348816,
-      "reward_std": 0.09432543441653252,
-      "rewards/tag_count_reward": 0.9635416865348816,
+      "loss": 0.0395,
+      "reward": 0.12500000558793545,
+      "reward_std": 0.2825479060411453,
+      "rewards/accuracy_reward": 0.12500000558793545,
       "step": 842
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 417.72918701171875,
+      "completion_length": 1987.0208740234375,
       "epoch": 0.843,
-      "grad_norm": 2.1216607471714966,
-      "kl": 0.21337890625,
+      "grad_norm": 13.751505977951913,
+      "kl": 1.01171875,
       "learning_rate": 1.659021412261026e-07,
-      "loss": 0.0097,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0273,
+      "reward": 0.1041666716337204,
+      "reward_std": 0.12873217463493347,
+      "rewards/accuracy_reward": 0.1041666716337204,
       "step": 843
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 534.6875305175781,
+      "completion_length": 2048.0,
       "epoch": 0.844,
-      "grad_norm": 34.203907822986416,
-      "kl": 1.27734375,
+      "grad_norm": 12.683337969666159,
+      "kl": 0.6640625,
       "learning_rate": 1.6508608292777203e-07,
-      "loss": 0.1112,
-      "reward": 0.96875,
-      "reward_std": 0.05653337761759758,
-      "rewards/tag_count_reward": 0.96875,
+      "loss": 0.0266,
+      "reward": 0.7291666865348816,
+      "reward_std": 0.43824970722198486,
+      "rewards/accuracy_reward": 0.7291666865348816,
       "step": 844
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 452.5208435058594,
+      "completion_length": 2048.0,
       "epoch": 0.845,
-      "grad_norm": 3.0906941460379644,
-      "kl": 0.2978515625,
+      "grad_norm": 7.800684132887189,
+      "kl": 1.15625,
       "learning_rate": 1.6427471468404952e-07,
-      "loss": 0.014,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0464,
+      "reward": 0.12500000558793545,
+      "reward_std": 0.2825479060411453,
+      "rewards/accuracy_reward": 0.12500000558793545,
       "step": 845
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 472.72918701171875,
+      "completion_length": 2048.0,
       "epoch": 0.846,
-      "grad_norm": 1.7592547700862164,
-      "kl": 0.3369140625,
+      "grad_norm": 14.909620719463177,
+      "kl": 0.681640625,
       "learning_rate": 1.6346804638120098e-07,
-      "loss": 0.007,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0272,
+      "reward": 0.7291666865348816,
+      "reward_std": 0.4565621018409729,
+      "rewards/accuracy_reward": 0.7291666865348816,
       "step": 846
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 465.62501525878906,
+      "completion_length": 2042.0833740234375,
       "epoch": 0.847,
-      "grad_norm": 1.8111958992652284,
-      "kl": 0.093017578125,
+      "grad_norm": 6.668131916637356,
+      "kl": 1.044921875,
       "learning_rate": 1.6266608784822542e-07,
-      "loss": 0.0012,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.041,
+      "reward": 0.1041666679084301,
+      "reward_std": 0.26679350435733795,
+      "rewards/accuracy_reward": 0.1041666679084301,
       "step": 847
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 494.85418701171875,
+      "completion_length": 2048.0,
       "epoch": 0.848,
-      "grad_norm": 2.2650504734811405,
-      "kl": 0.34521484375,
+      "grad_norm": 5.923338957864061,
+      "kl": 1.015625,
       "learning_rate": 1.6186884885673413e-07,
-      "loss": -0.004,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0406,
+      "reward": 0.3541666716337204,
+      "reward_std": 0.23615825921297073,
+      "rewards/accuracy_reward": 0.3541666716337204,
       "step": 848
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 428.0625,
+      "completion_length": 2048.0,
       "epoch": 0.849,
-      "grad_norm": 2.072160002401503,
-      "kl": 0.065673828125,
+      "grad_norm": 6.187418156758978,
+      "kl": 1.1796875,
       "learning_rate": 1.610763391208329e-07,
-      "loss": 0.016,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0471,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 849
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 431.9791717529297,
+      "completion_length": 2046.8333740234375,
       "epoch": 0.85,
-      "grad_norm": 3.2160210643606466,
-      "kl": 0.185546875,
+      "grad_norm": 9.063977633558439,
+      "kl": 0.7314453125,
       "learning_rate": 1.6028856829700258e-07,
-      "loss": 0.0443,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0293,
+      "reward": 0.5416666716337204,
+      "reward_std": 0.41310612857341766,
+      "rewards/accuracy_reward": 0.5416666716337204,
       "step": 850
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 471.625,
+      "completion_length": 2048.0,
       "epoch": 0.851,
-      "grad_norm": 1.6513631639591588,
-      "kl": 0.24169921875,
+      "grad_norm": 7.382357865670128,
+      "kl": 0.892578125,
       "learning_rate": 1.5950554598398228e-07,
-      "loss": 0.0202,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.024328090250492096,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0357,
+      "reward": 0.4166666716337204,
+      "reward_std": 0.26742906868457794,
+      "rewards/accuracy_reward": 0.4166666716337204,
       "step": 851
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 426.5833435058594,
+      "completion_length": 2014.5,
       "epoch": 0.852,
-      "grad_norm": 0.45549423537189326,
-      "kl": 0.081298828125,
+      "grad_norm": 11.152251535350036,
+      "kl": 0.712890625,
       "learning_rate": 1.5872728172265146e-07,
-      "loss": 0.0032,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0288,
+      "reward": 0.1875,
+      "reward_std": 0.19526028633117676,
+      "rewards/accuracy_reward": 0.1875,
       "step": 852
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 447.375,
+      "completion_length": 2048.0,
       "epoch": 0.853,
-      "grad_norm": 0.656777240615989,
-      "kl": 0.079345703125,
+      "grad_norm": 14.11874312053419,
+      "kl": 0.7080078125,
       "learning_rate": 1.579537849959148e-07,
-      "loss": 0.0035,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0283,
+      "reward": 0.5833333432674408,
+      "reward_std": 0.4056393951177597,
+      "rewards/accuracy_reward": 0.5833333432674408,
       "step": 853
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 466.2291717529297,
+      "completion_length": 2043.4583740234375,
       "epoch": 0.854,
-      "grad_norm": 2.2810028065560046,
-      "kl": 0.2255859375,
+      "grad_norm": 3.3242000278387325,
+      "kl": 1.10546875,
       "learning_rate": 1.5718506522858572e-07,
-      "loss": 0.0356,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0411,
+      "reward": 0.1666666716337204,
+      "reward_std": 0.12309150397777557,
+      "rewards/accuracy_reward": 0.1666666716337204,
       "step": 854
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 383.62501525878906,
+      "completion_length": 2016.8125,
       "epoch": 0.855,
-      "grad_norm": 1.6475080603537837,
-      "kl": 0.099853515625,
+      "grad_norm": 14.458312088661446,
+      "kl": 1.39453125,
       "learning_rate": 1.5642113178727193e-07,
-      "loss": 0.0046,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0543,
+      "reward": 0.125,
+      "reward_std": 0.13055823743343353,
+      "rewards/accuracy_reward": 0.125,
       "step": 855
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 438.8958435058594,
+      "completion_length": 2047.3333740234375,
       "epoch": 0.856,
-      "grad_norm": 5.468548790410516,
-      "kl": 0.2724609375,
+      "grad_norm": 9.719222044217002,
+      "kl": 0.865234375,
       "learning_rate": 1.5566199398026147e-07,
-      "loss": 0.0359,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0346,
+      "reward": 0.3541666865348816,
+      "reward_std": 0.3000393807888031,
+      "rewards/accuracy_reward": 0.3541666865348816,
       "step": 856
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 452.0416717529297,
+      "completion_length": 2031.4375,
       "epoch": 0.857,
-      "grad_norm": 0.44153691701559905,
-      "kl": 0.08203125,
+      "grad_norm": 17.639133803990116,
+      "kl": 0.904296875,
       "learning_rate": 1.5490766105740876e-07,
-      "loss": 0.0032,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0324,
+      "reward": 0.12500000558793545,
+      "reward_std": 0.20090095698833466,
+      "rewards/accuracy_reward": 0.12500000558793545,
       "step": 857
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 423.1458435058594,
+      "completion_length": 2048.0,
       "epoch": 0.858,
-      "grad_norm": 0.26511660327498243,
-      "kl": 0.0577392578125,
+      "grad_norm": 4.924928766566072,
+      "kl": 0.80078125,
       "learning_rate": 1.5415814221002265e-07,
-      "loss": 0.0023,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.032,
+      "reward": 0.06250000186264515,
+      "reward_std": 0.16948115825653076,
+      "rewards/accuracy_reward": 0.06250000186264515,
       "step": 858
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 476.5833435058594,
+      "completion_length": 2015.0208740234375,
       "epoch": 0.859,
-      "grad_norm": 3.1492680359957843,
-      "kl": 0.18505859375,
+      "grad_norm": 4.293035950258309,
+      "kl": 0.98046875,
       "learning_rate": 1.5341344657075354e-07,
-      "loss": 0.0432,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.05689104273915291,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0394,
+      "reward": 0.1875,
+      "reward_std": 0.11306675523519516,
+      "rewards/accuracy_reward": 0.1875,
       "step": 859
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 540.2083435058594,
+      "completion_length": 2048.0,
       "epoch": 0.86,
-      "grad_norm": 3.7943706513929647,
-      "kl": 0.2841796875,
+      "grad_norm": 3.2299913959999826,
+      "kl": 1.01953125,
       "learning_rate": 1.5267358321348285e-07,
-      "loss": 0.0567,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0409,
+      "reward": 0.1666666716337204,
+      "reward_std": 0.20090095698833466,
+      "rewards/accuracy_reward": 0.1666666716337204,
       "step": 860
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 497.3125,
+      "completion_length": 2015.0208740234375,
       "epoch": 0.861,
-      "grad_norm": 0.5809798704119035,
-      "kl": 0.089111328125,
+      "grad_norm": 3.2688304577631184,
+      "kl": 0.8828125,
       "learning_rate": 1.5193856115321224e-07,
-      "loss": 0.004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0283,
+      "reward": 0.3125000149011612,
+      "reward_std": 0.25182367861270905,
+      "rewards/accuracy_reward": 0.3125000149011612,
       "step": 861
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 413.5208435058594,
+      "completion_length": 2048.0,
       "epoch": 0.862,
-      "grad_norm": 0.2934335842153366,
-      "kl": 0.0626220703125,
+      "grad_norm": 2.6071571974017282,
+      "kl": 0.74609375,
       "learning_rate": 1.5120838934595337e-07,
-      "loss": 0.0026,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0298,
+      "reward": 0.2083333432674408,
+      "reward_std": 0.09731236100196838,
+      "rewards/accuracy_reward": 0.2083333432674408,
       "step": 862
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 470.3333435058594,
+      "completion_length": 2025.2083740234375,
       "epoch": 0.863,
-      "grad_norm": 1.5502893810310887,
-      "kl": 0.068115234375,
+      "grad_norm": 2.7169747923674263,
+      "kl": 1.001953125,
       "learning_rate": 1.5048307668861947e-07,
-      "loss": -0.0,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0444,
+      "reward": 0.1666666716337204,
+      "reward_std": 0.20090095698833466,
+      "rewards/accuracy_reward": 0.1666666716337204,
       "step": 863
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 416.5416717529297,
+      "completion_length": 1996.7708740234375,
       "epoch": 0.864,
-      "grad_norm": 0.558178976097742,
-      "kl": 0.080810546875,
+      "grad_norm": 4.699981490991396,
+      "kl": 0.77734375,
       "learning_rate": 1.4976263201891613e-07,
-      "loss": 0.0036,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0369,
+      "reward": 0.125,
+      "reward_std": 0.13055823743343353,
+      "rewards/accuracy_reward": 0.125,
       "step": 864
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 441.29168701171875,
+      "completion_length": 1994.6666870117188,
       "epoch": 0.865,
-      "grad_norm": 3.341005370129586,
-      "kl": 0.14990234375,
+      "grad_norm": 3.9300914504019624,
+      "kl": 0.970703125,
       "learning_rate": 1.4904706411523448e-07,
-      "loss": -0.0057,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": 0.0382,
+      "reward": 0.1041666716337204,
+      "reward_std": 0.12873217463493347,
+      "rewards/accuracy_reward": 0.1041666716337204,
       "step": 865
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 467.2291717529297,
+      "completion_length": 2008.5833740234375,
       "epoch": 0.866,
-      "grad_norm": 5.232549834844898,
-      "kl": 0.2735595703125,
+      "grad_norm": 6.5956826280239005,
+      "kl": 0.65234375,
       "learning_rate": 1.483363816965435e-07,
-      "loss": 0.1932,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0318,
+      "reward": 0.2083333395421505,
+      "reward_std": 0.26742906868457794,
+      "rewards/accuracy_reward": 0.2083333395421505,
       "step": 866
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 455.97918701171875,
+      "completion_length": 1993.9375610351562,
       "epoch": 0.867,
-      "grad_norm": 3.482765515194214,
-      "kl": 0.0701904296875,
+      "grad_norm": 2.5936118939689097,
+      "kl": 0.671875,
       "learning_rate": 1.4763059342228434e-07,
-      "loss": 0.0014,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.07216878235340118,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0256,
+      "reward": 0.1875,
+      "reward_std": 0.19526028633117676,
+      "rewards/accuracy_reward": 0.1875,
       "step": 867
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 437.3125,
+      "completion_length": 2048.0,
       "epoch": 0.868,
-      "grad_norm": 3.6447899326522513,
-      "kl": 0.25732421875,
+      "grad_norm": 3.0562317150803424,
+      "kl": 0.802734375,
       "learning_rate": 1.469297078922642e-07,
-      "loss": 0.038,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0321,
+      "reward": 0.16666667722165585,
+      "reward_std": 0.20090095698833466,
+      "rewards/accuracy_reward": 0.16666667722165585,
       "step": 868
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 392.68751525878906,
+      "completion_length": 1979.5,
       "epoch": 0.869,
-      "grad_norm": 2.4560052168914495,
-      "kl": 0.0963134765625,
+      "grad_norm": 1.5069183640892052,
+      "kl": 0.716796875,
       "learning_rate": 1.4623373364655223e-07,
-      "loss": -0.0033,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0293,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 869
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 423.3958435058594,
+      "completion_length": 2009.7916870117188,
       "epoch": 0.87,
-      "grad_norm": 0.29718128547812317,
-      "kl": 0.04736328125,
+      "grad_norm": 4.3524794092839665,
+      "kl": 0.66796875,
       "learning_rate": 1.4554267916537495e-07,
-      "loss": 0.0021,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.027,
+      "reward": 0.1875000111758709,
+      "reward_std": 0.3470645844936371,
+      "rewards/accuracy_reward": 0.1875000111758709,
       "step": 870
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 431.12501525878906,
+      "completion_length": 2048.0,
       "epoch": 0.871,
-      "grad_norm": 4.346343773847815,
-      "kl": 0.19873046875,
+      "grad_norm": 3.1364849765515275,
+      "kl": 0.880859375,
       "learning_rate": 1.448565528690129e-07,
-      "loss": 0.0173,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0353,
+      "reward": 0.06250000186264515,
+      "reward_std": 0.16948114335536957,
+      "rewards/accuracy_reward": 0.06250000186264515,
       "step": 871
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 400.8333435058594,
+      "completion_length": 2029.125,
       "epoch": 0.872,
-      "grad_norm": 0.3860896818957767,
-      "kl": 0.05810546875,
+      "grad_norm": 6.045943405482593,
+      "kl": 0.654296875,
       "learning_rate": 1.4417536311769885e-07,
-      "loss": 0.0027,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.025,
+      "reward": 0.1875000111758709,
+      "reward_std": 0.3804958164691925,
+      "rewards/accuracy_reward": 0.1875000111758709,
       "step": 872
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 453.75001525878906,
+      "completion_length": 2048.0,
       "epoch": 0.873,
-      "grad_norm": 2.6455563018916184,
-      "kl": 0.0634765625,
+      "grad_norm": 1.8268370794202256,
+      "kl": 0.78515625,
       "learning_rate": 1.4349911821151462e-07,
-      "loss": 0.0052,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0314,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 873
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 462.4791717529297,
+      "completion_length": 2048.0,
       "epoch": 0.874,
-      "grad_norm": 5.504596501408147,
-      "kl": 0.4541015625,
+      "grad_norm": 1.4342003494389164,
+      "kl": 0.6826171875,
       "learning_rate": 1.4282782639029128e-07,
-      "loss": 0.003,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0273,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 874
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 469.7500305175781,
+      "completion_length": 2015.8958740234375,
       "epoch": 0.875,
-      "grad_norm": 3.488122009899489,
-      "kl": 0.1461181640625,
+      "grad_norm": 1.634441378463937,
+      "kl": 0.630859375,
       "learning_rate": 1.4216149583350755e-07,
-      "loss": 0.0022,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.07216878235340118,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0254,
+      "reward": 0.1666666716337204,
+      "reward_std": 0.24618300795555115,
+      "rewards/accuracy_reward": 0.1666666716337204,
       "step": 875
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 467.31251525878906,
+      "completion_length": 2048.0,
       "epoch": 0.876,
-      "grad_norm": 2.652800552168764,
-      "kl": 0.21240234375,
+      "grad_norm": 1.4334835435780557,
+      "kl": 0.615234375,
       "learning_rate": 1.4150013466019114e-07,
-      "loss": 0.0464,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.04865618050098419,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0246,
+      "reward": 0.0625,
+      "reward_std": 0.16948114335536957,
+      "rewards/accuracy_reward": 0.0625,
       "step": 876
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 405.1041717529297,
+      "completion_length": 2048.0,
       "epoch": 0.877,
-      "grad_norm": 2.384109778280924,
-      "kl": 0.12841796875,
+      "grad_norm": 0.6215863651095573,
+      "kl": 0.658203125,
       "learning_rate": 1.4084375092881917e-07,
-      "loss": 0.0122,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0263,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 877
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 493.81251525878906,
+      "completion_length": 2048.0,
       "epoch": 0.878,
-      "grad_norm": 1.833720375747527,
-      "kl": 0.16552734375,
+      "grad_norm": 2.3010624211097634,
+      "kl": 0.5859375,
       "learning_rate": 1.4019235263722034e-07,
-      "loss": 0.0264,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0234,
+      "reward": 0.08333333395421505,
+      "reward_std": 0.24164992570877075,
+      "rewards/accuracy_reward": 0.08333333395421505,
       "step": 878
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 463.2916717529297,
+      "completion_length": 2048.0,
       "epoch": 0.879,
-      "grad_norm": 7.251525232629622,
-      "kl": 0.595703125,
+      "grad_norm": 0.6648640432402617,
+      "kl": 0.62890625,
       "learning_rate": 1.395459477224772e-07,
-      "loss": 0.0301,
-      "reward": 1.0,
+      "loss": 0.0251,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 879
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 452.0208435058594,
+      "completion_length": 2048.0,
       "epoch": 0.88,
-      "grad_norm": 1.0364271720065523,
-      "kl": 0.122802734375,
+      "grad_norm": 2.0236126555758682,
+      "kl": 0.853515625,
       "learning_rate": 1.3890454406082956e-07,
-      "loss": 0.0057,
-      "reward": 1.0,
+      "loss": 0.0341,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 880
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 412.5833435058594,
+      "completion_length": 2009.4375,
       "epoch": 0.881,
-      "grad_norm": 9.909144868862827,
-      "kl": 0.8623046875,
+      "grad_norm": 1.128476162151685,
+      "kl": 0.66015625,
       "learning_rate": 1.3826814946757888e-07,
-      "loss": 0.0001,
-      "reward": 0.9895833730697632,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833730697632,
+      "loss": 0.0268,
+      "reward": 0.0833333358168602,
+      "reward_std": 0.18523554503917694,
+      "rewards/accuracy_reward": 0.0833333358168602,
       "step": 881
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 499.16668701171875,
+      "completion_length": 2048.0,
       "epoch": 0.882,
-      "grad_norm": 11.477329590363407,
-      "kl": 0.501953125,
+      "grad_norm": 1.0489429549652802,
+      "kl": 0.69140625,
       "learning_rate": 1.3763677169699217e-07,
-      "loss": 0.337,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": 0.0277,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.09731236100196838,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 882
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 416.22918701171875,
+      "completion_length": 2009.0416870117188,
       "epoch": 0.883,
-      "grad_norm": 2.2472709963991817,
-      "kl": 0.16015625,
+      "grad_norm": 1.6157651362270744,
+      "kl": 0.68359375,
       "learning_rate": 1.370104184422085e-07,
-      "loss": 0.0062,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0927,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 883
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 541.6666870117188,
+      "completion_length": 2011.7291870117188,
       "epoch": 0.884,
-      "grad_norm": 5.7416781883628625,
-      "kl": 0.939453125,
+      "grad_norm": 1.2647522711797208,
+      "kl": 0.607421875,
       "learning_rate": 1.3638909733514452e-07,
-      "loss": 0.1641,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.024328090250492096,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0848,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 884
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 419.1458435058594,
+      "completion_length": 2048.0,
       "epoch": 0.885,
-      "grad_norm": 2.5819552151406615,
-      "kl": 0.236328125,
+      "grad_norm": 1.8606034069346222,
+      "kl": 0.630859375,
       "learning_rate": 1.3577281594640182e-07,
-      "loss": 0.0105,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0252,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 885
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 390.0833435058594,
+      "completion_length": 2048.0,
       "epoch": 0.886,
-      "grad_norm": 2.2738960276281825,
-      "kl": 0.17041015625,
+      "grad_norm": 1.6401433060203334,
+      "kl": 0.68359375,
       "learning_rate": 1.351615817851748e-07,
-      "loss": 0.0277,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0273,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 886
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 453.18751525878906,
+      "completion_length": 2048.0,
       "epoch": 0.887,
-      "grad_norm": 2.318452927213316,
-      "kl": 0.369140625,
+      "grad_norm": 1.1263058323581188,
+      "kl": 0.6015625,
       "learning_rate": 1.345554022991586e-07,
-      "loss": 0.0422,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0241,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 887
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 440.0416717529297,
+      "completion_length": 2048.0,
       "epoch": 0.888,
-      "grad_norm": 0.9147712689098717,
-      "kl": 0.130615234375,
+      "grad_norm": 0.9803981112851079,
+      "kl": 0.552734375,
       "learning_rate": 1.3395428487445914e-07,
-      "loss": 0.0056,
-      "reward": 1.0,
+      "loss": 0.0222,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 888
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 360.93751525878906,
+      "completion_length": 2048.0,
       "epoch": 0.889,
-      "grad_norm": 2.337766683078408,
-      "kl": 0.178466796875,
+      "grad_norm": 1.0037521575742177,
+      "kl": 0.580078125,
       "learning_rate": 1.3335823683550237e-07,
-      "loss": 0.0235,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0232,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 889
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 454.56251525878906,
+      "completion_length": 2048.0,
       "epoch": 0.89,
-      "grad_norm": 1.726684676479182,
-      "kl": 0.2177734375,
+      "grad_norm": 0.4832843221227563,
+      "kl": 0.4794921875,
       "learning_rate": 1.3276726544494571e-07,
-      "loss": 0.0089,
-      "reward": 1.0,
+      "loss": 0.0191,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 890
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 494.18751525878906,
+      "completion_length": 2045.5208740234375,
       "epoch": 0.891,
-      "grad_norm": 7.304317674135185,
-      "kl": 0.42041015625,
+      "grad_norm": 1.4733271132589247,
+      "kl": 0.60546875,
       "learning_rate": 1.3218137790358892e-07,
-      "loss": 0.1586,
-      "reward": 0.984375,
-      "reward_std": 0.03884884715080261,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0242,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 891
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 448.56251525878906,
+      "completion_length": 2048.0,
       "epoch": 0.892,
-      "grad_norm": 0.442336781880095,
-      "kl": 0.08056640625,
+      "grad_norm": 1.5110346456887616,
+      "kl": 0.478515625,
       "learning_rate": 1.316005813502869e-07,
-      "loss": 0.0032,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0191,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 892
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 410.66668701171875,
+      "completion_length": 2010.5625,
       "epoch": 0.893,
-      "grad_norm": 0.5853659596469007,
-      "kl": 0.0888671875,
+      "grad_norm": 1.5550203560960716,
+      "kl": 0.439453125,
       "learning_rate": 1.3102488286186234e-07,
-      "loss": 0.0039,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.077,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.14433756470680237,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 893
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 423.5208435058594,
+      "completion_length": 2009.3125,
       "epoch": 0.894,
-      "grad_norm": 9.054535838277454,
-      "kl": 0.546875,
+      "grad_norm": 1.4645912112434158,
+      "kl": 0.443359375,
       "learning_rate": 1.3045428945301953e-07,
-      "loss": 0.0423,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0179,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 894
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 359.9583435058594,
+      "completion_length": 2044.8958740234375,
       "epoch": 0.895,
-      "grad_norm": 4.654534141804648,
-      "kl": 0.08447265625,
+      "grad_norm": 2.0399190537422394,
+      "kl": 0.40625,
       "learning_rate": 1.2988880807625927e-07,
-      "loss": 0.0139,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0158,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 895
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 418.7083435058594,
+      "completion_length": 2048.0,
       "epoch": 0.896,
-      "grad_norm": 4.214225314069966,
-      "kl": 0.294677734375,
+      "grad_norm": 0.7139205554757724,
+      "kl": 0.451171875,
       "learning_rate": 1.2932844562179352e-07,
-      "loss": 0.0797,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.018,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 896
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 398.3958435058594,
+      "completion_length": 2048.0,
       "epoch": 0.897,
-      "grad_norm": 2.90752008700908,
-      "kl": 0.17138671875,
+      "grad_norm": 0.41456375808433443,
+      "kl": 0.4287109375,
       "learning_rate": 1.2877320891746201e-07,
-      "loss": -0.0046,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0172,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 897
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 420.0833435058594,
+      "completion_length": 2048.0,
       "epoch": 0.898,
-      "grad_norm": 0.47940129933285197,
-      "kl": 0.07275390625,
+      "grad_norm": 0.4426278980066036,
+      "kl": 0.4609375,
       "learning_rate": 1.2822310472864885e-07,
-      "loss": 0.0032,
-      "reward": 1.0,
+      "loss": 0.0184,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 898
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 467.6458435058594,
+      "completion_length": 2011.7708740234375,
       "epoch": 0.899,
-      "grad_norm": 2.3433899793970165,
-      "kl": 0.1708984375,
+      "grad_norm": 1.2764995279998117,
+      "kl": 0.4130859375,
       "learning_rate": 1.2767813975819983e-07,
-      "loss": 0.0115,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0166,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 899
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 465.4583435058594,
+      "completion_length": 2048.0,
       "epoch": 0.9,
-      "grad_norm": 8.787946894065907,
-      "kl": 0.46923828125,
+      "grad_norm": 2.413689301188002,
+      "kl": 0.4765625,
       "learning_rate": 1.2713832064634125e-07,
-      "loss": 0.104,
-      "reward": 0.9739583432674408,
-      "reward_std": 0.0749332457780838,
-      "rewards/tag_count_reward": 0.9739583432674408,
+      "loss": 0.019,
+      "reward": 0.18750000558793545,
+      "reward_std": 0.3000393956899643,
+      "rewards/accuracy_reward": 0.18750000558793545,
       "step": 900
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 441.9583435058594,
+      "completion_length": 2048.0,
       "epoch": 0.901,
-      "grad_norm": 1.9942745799790182,
-      "kl": 0.19189453125,
+      "grad_norm": 1.7507374776945797,
+      "kl": 0.470703125,
       "learning_rate": 1.2660365397059856e-07,
-      "loss": 0.0118,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0189,
+      "reward": 0.1666666716337204,
+      "reward_std": 0.24179892987012863,
+      "rewards/accuracy_reward": 0.1666666716337204,
       "step": 901
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 486.6458435058594,
+      "completion_length": 2048.0,
       "epoch": 0.902,
-      "grad_norm": 1.4209046426026215,
-      "kl": 0.152587890625,
+      "grad_norm": 1.8289861388933655,
+      "kl": 0.4560546875,
       "learning_rate": 1.260741462457165e-07,
-      "loss": 0.0074,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0182,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.09731236100196838,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 902
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 411.00001525878906,
+      "completion_length": 2048.0,
       "epoch": 0.903,
-      "grad_norm": 3.3172086388415796,
-      "kl": 0.2999267578125,
+      "grad_norm": 2.7962622655589735,
+      "kl": 0.455078125,
       "learning_rate": 1.2554980392357956e-07,
-      "loss": 0.0598,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0182,
+      "reward": 0.0833333358168602,
+      "reward_std": 0.19462472200393677,
+      "rewards/accuracy_reward": 0.0833333358168602,
       "step": 903
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 491.4375305175781,
+      "completion_length": 2041.2291870117188,
       "epoch": 0.904,
-      "grad_norm": 5.075284253325144,
-      "kl": 0.5068359375,
+      "grad_norm": 1.4445803772770263,
+      "kl": 0.3994140625,
       "learning_rate": 1.2503063339313356e-07,
-      "loss": 0.1422,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.016,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 904
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 433.1041717529297,
+      "completion_length": 2048.0,
       "epoch": 0.905,
-      "grad_norm": 4.894805146692723,
-      "kl": 0.38818359375,
+      "grad_norm": 0.5582190257259914,
+      "kl": 0.41796875,
       "learning_rate": 1.2451664098030743e-07,
-      "loss": 0.0244,
-      "reward": 1.0,
+      "loss": 0.0167,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 905
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 433.00001525878906,
+      "completion_length": 2013.4375610351562,
       "epoch": 0.906,
-      "grad_norm": 1.6623425457072913,
-      "kl": 0.134521484375,
+      "grad_norm": 2.279879222494776,
+      "kl": 0.4580078125,
       "learning_rate": 1.2400783294793668e-07,
-      "loss": 0.0214,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0185,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 906
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 466.18751525878906,
+      "completion_length": 2048.0,
       "epoch": 0.907,
-      "grad_norm": 4.517468250608921,
-      "kl": 0.296875,
+      "grad_norm": 1.575166697236276,
+      "kl": 0.4443359375,
       "learning_rate": 1.235042154956865e-07,
-      "loss": 0.0592,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.024328090250492096,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0177,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 907
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 466.375,
+      "completion_length": 2048.0,
       "epoch": 0.908,
-      "grad_norm": 1.9574484289394507,
-      "kl": 0.18359375,
+      "grad_norm": 3.0639734414681357,
+      "kl": 0.4345703125,
       "learning_rate": 1.2300579475997657e-07,
-      "loss": 0.0107,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0174,
+      "reward": 0.20833333395421505,
+      "reward_std": 0.32399244606494904,
+      "rewards/accuracy_reward": 0.20833333395421505,
       "step": 908
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 470.8333435058594,
+      "completion_length": 2048.0,
       "epoch": 0.909,
-      "grad_norm": 0.5829037579667905,
-      "kl": 0.078125,
+      "grad_norm": 0.5697167671132396,
+      "kl": 0.443359375,
       "learning_rate": 1.2251257681390645e-07,
-      "loss": 0.0039,
-      "reward": 1.0,
+      "loss": 0.0177,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 909
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 452.35418701171875,
+      "completion_length": 2048.0,
       "epoch": 0.91,
-      "grad_norm": 2.3185634515942803,
-      "kl": 0.314453125,
+      "grad_norm": 0.656173164490481,
+      "kl": 0.4208984375,
       "learning_rate": 1.220245676671809e-07,
-      "loss": 0.0325,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.05689104273915291,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0169,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 910
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 410.6666717529297,
+      "completion_length": 2008.1875,
       "epoch": 0.911,
-      "grad_norm": 4.990915064622496,
-      "kl": 0.18603515625,
+      "grad_norm": 0.49772025484301624,
+      "kl": 0.3701171875,
       "learning_rate": 1.2154177326603763e-07,
-      "loss": 0.0313,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0148,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 911
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 458.22918701171875,
+      "completion_length": 2048.0,
       "epoch": 0.912,
-      "grad_norm": 0.9703749274233409,
-      "kl": 0.1337890625,
+      "grad_norm": 1.3051873148988706,
+      "kl": 0.365234375,
       "learning_rate": 1.2106419949317388e-07,
-      "loss": 0.0056,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0146,
+      "reward": 0.0833333358168602,
+      "reward_std": 0.12309150397777557,
+      "rewards/accuracy_reward": 0.0833333358168602,
       "step": 912
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 388.3541717529297,
+      "completion_length": 2048.0,
       "epoch": 0.913,
-      "grad_norm": 3.0648569871424067,
-      "kl": 0.1068115234375,
+      "grad_norm": 1.5064562908799253,
+      "kl": 0.3369140625,
       "learning_rate": 1.2059185216767543e-07,
-      "loss": 0.0401,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0135,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 913
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 467.75,
+      "completion_length": 2048.0,
       "epoch": 0.914,
-      "grad_norm": 4.927041693824767,
-      "kl": 0.495849609375,
+      "grad_norm": 0.8898969887221723,
+      "kl": 0.3486328125,
       "learning_rate": 1.2012473704494537e-07,
-      "loss": 0.0826,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0139,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 914
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 389.6458435058594,
+      "completion_length": 2048.0,
       "epoch": 0.915,
-      "grad_norm": 0.36372839672372403,
-      "kl": 0.0556640625,
+      "grad_norm": 0.3419120657078177,
+      "kl": 0.3291015625,
       "learning_rate": 1.1966285981663407e-07,
-      "loss": 0.0026,
-      "reward": 1.0,
+      "loss": 0.0132,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 915
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 432.375,
+      "completion_length": 2048.0,
       "epoch": 0.916,
-      "grad_norm": 2.696301347232995,
-      "kl": 0.184814453125,
+      "grad_norm": 0.7998588620427595,
+      "kl": 0.353515625,
       "learning_rate": 1.1920622611056974e-07,
-      "loss": 0.0306,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0141,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 916
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 419.4791717529297,
+      "completion_length": 2009.7083740234375,
       "epoch": 0.917,
-      "grad_norm": 3.225344987717421,
-      "kl": 0.382568359375,
+      "grad_norm": 0.9581409748480985,
+      "kl": 0.3095703125,
       "learning_rate": 1.1875484149069004e-07,
-      "loss": 0.0374,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0125,
+      "reward": 0.0625,
+      "reward_std": 0.11306675523519516,
+      "rewards/accuracy_reward": 0.0625,
       "step": 917
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 494.0416717529297,
+      "completion_length": 2048.0,
       "epoch": 0.918,
-      "grad_norm": 1.2396993112021069,
-      "kl": 0.177001953125,
+      "grad_norm": 1.3201177912116782,
+      "kl": 0.322265625,
       "learning_rate": 1.1830871145697412e-07,
-      "loss": 0.0078,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0129,
+      "reward": 0.0625,
+      "reward_std": 0.11306675523519516,
+      "rewards/accuracy_reward": 0.0625,
       "step": 918
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 427.54168701171875,
+      "completion_length": 2048.0,
       "epoch": 0.919,
-      "grad_norm": 2.675512597659298,
-      "kl": 0.29833984375,
+      "grad_norm": 0.28453430764005744,
+      "kl": 0.3017578125,
       "learning_rate": 1.1786784144537563e-07,
-      "loss": 0.0868,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0121,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 919
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 429.8958435058594,
+      "completion_length": 2048.0,
       "epoch": 0.92,
-      "grad_norm": 0.5774129184213032,
-      "kl": 0.07568359375,
+      "grad_norm": 0.32623963840603776,
+      "kl": 0.3251953125,
       "learning_rate": 1.1743223682775649e-07,
-      "loss": 0.0033,
-      "reward": 1.0,
+      "loss": 0.013,
+      "reward": 0.0,
       "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 920
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 448.0208435058594,
+      "completion_length": 2048.0,
       "epoch": 0.921,
-      "grad_norm": 2.1214438632925203,
-      "kl": 0.25,
+      "grad_norm": 2.7277000105199627,
+      "kl": 0.27734375,
       "learning_rate": 1.1700190291182158e-07,
-      "loss": -0.0132,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0111,
+      "reward": 0.06250000186264515,
+      "reward_std": 0.16948114335536957,
+      "rewards/accuracy_reward": 0.06250000186264515,
       "step": 921
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 492.10418701171875,
+      "completion_length": 2048.0,
       "epoch": 0.922,
-      "grad_norm": 1.9715037166003222,
-      "kl": 0.11474609375,
+      "grad_norm": 1.9537063728810338,
+      "kl": 0.3017578125,
       "learning_rate": 1.1657684494105386e-07,
-      "loss": 0.0149,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0121,
+      "reward": 0.06250000186264515,
+      "reward_std": 0.16948114335536957,
+      "rewards/accuracy_reward": 0.06250000186264515,
       "step": 922
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 463.3333435058594,
+      "completion_length": 2010.2916870117188,
       "epoch": 0.923,
-      "grad_norm": 2.997274956666607,
-      "kl": 0.249267578125,
+      "grad_norm": 0.3716521139561905,
+      "kl": 0.30078125,
       "learning_rate": 1.1615706809465051e-07,
-      "loss": -0.0124,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0121,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 923
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 458.1458435058594,
+      "completion_length": 2048.0,
       "epoch": 0.924,
-      "grad_norm": 2.4679882370570208,
-      "kl": 0.216064453125,
+      "grad_norm": 3.335435078556315,
+      "kl": 0.2919921875,
       "learning_rate": 1.1574257748745986e-07,
-      "loss": 0.0178,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0117,
+      "reward": 0.1041666716337204,
+      "reward_std": 0.19526028633117676,
+      "rewards/accuracy_reward": 0.1041666716337204,
       "step": 924
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 425.50001525878906,
+      "completion_length": 2048.0,
       "epoch": 0.925,
-      "grad_norm": 2.788486659392232,
-      "kl": 0.170166015625,
+      "grad_norm": 0.26394143098604256,
+      "kl": 0.2919921875,
       "learning_rate": 1.1533337816991931e-07,
-      "loss": 0.038,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0117,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 925
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 448.25,
+      "completion_length": 1973.9166870117188,
       "epoch": 0.926,
-      "grad_norm": 4.4389011117909565,
-      "kl": 0.376220703125,
+      "grad_norm": 1.9226438487285717,
+      "kl": 0.3017578125,
       "learning_rate": 1.1492947512799328e-07,
-      "loss": 0.0454,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": -0.0168,
+      "reward": 0.1458333432674408,
+      "reward_std": 0.12873217463493347,
+      "rewards/accuracy_reward": 0.1458333432674408,
       "step": 926
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 419.81251525878906,
+      "completion_length": 2048.0,
       "epoch": 0.927,
-      "grad_norm": 0.3004188751162577,
-      "kl": 0.060546875,
+      "grad_norm": 2.1356497514478185,
+      "kl": 0.3125,
       "learning_rate": 1.1453087328311299e-07,
-      "loss": 0.0026,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0125,
+      "reward": 0.06250000186264515,
+      "reward_std": 0.16948114335536957,
+      "rewards/accuracy_reward": 0.06250000186264515,
       "step": 927
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 463.68751525878906,
+      "completion_length": 1983.2083740234375,
       "epoch": 0.928,
-      "grad_norm": 0.5861374247940305,
-      "kl": 0.09130859375,
+      "grad_norm": 2.877170336108542,
+      "kl": 0.3193359375,
       "learning_rate": 1.1413757749211602e-07,
-      "loss": 0.004,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0147,
+      "reward": 0.229166679084301,
+      "reward_std": 0.25182367861270905,
+      "rewards/accuracy_reward": 0.229166679084301,
       "step": 928
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 458.29168701171875,
+      "completion_length": 2048.0,
       "epoch": 0.929,
-      "grad_norm": 0.22493358475542433,
-      "kl": 0.0609130859375,
+      "grad_norm": 1.7908075357104538,
+      "kl": 0.3125,
       "learning_rate": 1.137495925471875e-07,
-      "loss": 0.0025,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0125,
+      "reward": 0.06250000186264515,
+      "reward_std": 0.16948114335536957,
+      "rewards/accuracy_reward": 0.06250000186264515,
       "step": 929
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 468.3958435058594,
+      "completion_length": 2041.7083740234375,
       "epoch": 0.93,
-      "grad_norm": 0.3975409634684941,
-      "kl": 0.079833984375,
+      "grad_norm": 1.1259554235534162,
+      "kl": 0.328125,
       "learning_rate": 1.1336692317580158e-07,
-      "loss": 0.0034,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0107,
+      "reward": 0.0833333358168602,
+      "reward_std": 0.12309150397777557,
+      "rewards/accuracy_reward": 0.0833333358168602,
       "step": 930
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 454.5833435058594,
+      "completion_length": 2048.0,
       "epoch": 0.931,
-      "grad_norm": 0.7831927134076526,
-      "kl": 0.11474609375,
+      "grad_norm": 3.5080913194814283,
+      "kl": 0.349609375,
       "learning_rate": 1.1298957404066381e-07,
-      "loss": 0.005,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.014,
+      "reward": 0.2916666679084301,
+      "reward_std": 0.3547767102718353,
+      "rewards/accuracy_reward": 0.2916666679084301,
       "step": 931
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 453.79168701171875,
+      "completion_length": 2048.0,
       "epoch": 0.932,
-      "grad_norm": 0.2565546048580399,
-      "kl": 0.0589599609375,
+      "grad_norm": 1.7446845620264881,
+      "kl": 0.328125,
       "learning_rate": 1.1261754973965422e-07,
-      "loss": 0.0024,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0131,
+      "reward": 0.2083333395421505,
+      "reward_std": 0.22040386497974396,
+      "rewards/accuracy_reward": 0.2083333395421505,
       "step": 932
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 483.3125305175781,
+      "completion_length": 2048.0,
       "epoch": 0.933,
-      "grad_norm": 5.965932854238622,
-      "kl": 0.3040771484375,
+      "grad_norm": 0.2995834088057952,
+      "kl": 0.3037109375,
       "learning_rate": 1.1225085480577158e-07,
-      "loss": 0.0664,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0122,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 933
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 401.2083435058594,
+      "completion_length": 2042.1666870117188,
       "epoch": 0.934,
-      "grad_norm": 0.5902863758628627,
-      "kl": 0.10260009765625,
+      "grad_norm": 1.4601731713262884,
+      "kl": 0.3232421875,
       "learning_rate": 1.1188949370707787e-07,
-      "loss": 0.0042,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0129,
+      "reward": 0.0625,
+      "reward_std": 0.11306675523519516,
+      "rewards/accuracy_reward": 0.0625,
       "step": 934
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 417.50001525878906,
+      "completion_length": 2037.9375,
       "epoch": 0.935,
-      "grad_norm": 0.27583183911270276,
-      "kl": 0.0458984375,
+      "grad_norm": 1.4633475182482905,
+      "kl": 0.3310546875,
       "learning_rate": 1.1153347084664419e-07,
-      "loss": 0.0024,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0091,
+      "reward": 0.16666667722165585,
+      "reward_std": 0.20090095698833466,
+      "rewards/accuracy_reward": 0.16666667722165585,
       "step": 935
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 433.2291717529297,
+      "completion_length": 2010.0208740234375,
       "epoch": 0.936,
-      "grad_norm": 1.666916279660374,
-      "kl": 0.111572265625,
+      "grad_norm": 2.8339153430231763,
+      "kl": 0.3154296875,
       "learning_rate": 1.1118279056249653e-07,
-      "loss": -0.0118,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": -0.0102,
+      "reward": 0.2916666716337204,
+      "reward_std": 0.3314591944217682,
+      "rewards/accuracy_reward": 0.2916666716337204,
       "step": 936
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 393.9166717529297,
+      "completion_length": 2038.2083740234375,
       "epoch": 0.937,
-      "grad_norm": 1.3125799330948598,
-      "kl": 0.0460205078125,
+      "grad_norm": 1.4499959770441926,
+      "kl": 0.322265625,
       "learning_rate": 1.1083745712756364e-07,
-      "loss": -0.0065,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0129,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 937
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 438.9583435058594,
+      "completion_length": 2006.25,
       "epoch": 0.938,
-      "grad_norm": 2.4309526964560475,
-      "kl": 0.089599609375,
+      "grad_norm": 1.3665074836865239,
+      "kl": 0.3330078125,
       "learning_rate": 1.1049747474962444e-07,
-      "loss": 0.0167,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.024328090250492096,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0525,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.09731236100196838,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 938
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 403.7083435058594,
+      "completion_length": 2008.9583740234375,
       "epoch": 0.939,
-      "grad_norm": 2.0975552361291014,
-      "kl": 0.1544189453125,
+      "grad_norm": 2.4353230165377586,
+      "kl": 0.357421875,
       "learning_rate": 1.1016284757125685e-07,
-      "loss": 0.0053,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.024328093975782394,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0144,
+      "reward": 0.1875000111758709,
+      "reward_std": 0.22604453563690186,
+      "rewards/accuracy_reward": 0.1875000111758709,
       "step": 939
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 454.29168701171875,
+      "completion_length": 2045.8125,
       "epoch": 0.94,
-      "grad_norm": 4.360268899437786,
-      "kl": 0.097412109375,
+      "grad_norm": 2.1396216036944233,
+      "kl": 0.33984375,
       "learning_rate": 1.0983357966978745e-07,
-      "loss": 0.0512,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.07216878235340118,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0136,
+      "reward": 0.1666666716337204,
+      "reward_std": 0.2278706133365631,
+      "rewards/accuracy_reward": 0.1666666716337204,
       "step": 940
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 405.1458435058594,
+      "completion_length": 1938.6041870117188,
       "epoch": 0.941,
-      "grad_norm": 0.35462463024772617,
-      "kl": 0.0648193359375,
+      "grad_norm": 2.835839147232048,
+      "kl": 0.314453125,
       "learning_rate": 1.0950967505724175e-07,
-      "loss": 0.0029,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.055,
+      "reward": 0.229166679084301,
+      "reward_std": 0.3139677196741104,
+      "rewards/accuracy_reward": 0.229166679084301,
       "step": 941
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 483.66668701171875,
+      "completion_length": 2010.8333740234375,
       "epoch": 0.942,
-      "grad_norm": 5.991695051097315,
-      "kl": 0.1943359375,
+      "grad_norm": 1.552318658456,
+      "kl": 0.3212890625,
       "learning_rate": 1.0919113768029517e-07,
-      "loss": 0.0394,
-      "reward": 0.9739583730697632,
-      "reward_std": 0.09021097794175148,
-      "rewards/tag_count_reward": 0.9739583730697632,
+      "loss": 0.0129,
+      "reward": 0.0625,
+      "reward_std": 0.11306675523519516,
+      "rewards/accuracy_reward": 0.0625,
       "step": 942
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 418.2083435058594,
+      "completion_length": 2044.541748046875,
       "epoch": 0.943,
-      "grad_norm": 0.23605136486972625,
-      "kl": 0.053955078125,
+      "grad_norm": 1.8860748871035848,
+      "kl": 0.3720703125,
       "learning_rate": 1.0887797142022521e-07,
-      "loss": 0.0024,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0147,
+      "reward": 0.0625,
+      "reward_std": 0.11306675523519516,
+      "rewards/accuracy_reward": 0.0625,
       "step": 943
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 461.6875,
+      "completion_length": 2035.7291870117188,
       "epoch": 0.944,
-      "grad_norm": 3.0034285302091157,
-      "kl": 0.3358154296875,
+      "grad_norm": 1.4478916180118966,
+      "kl": 0.3154296875,
       "learning_rate": 1.0857018009286381e-07,
-      "loss": 0.0398,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.009,
+      "reward": 0.16666667722165585,
+      "reward_std": 0.20090095698833466,
+      "rewards/accuracy_reward": 0.16666667722165585,
       "step": 944
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 441.6458435058594,
+      "completion_length": 1991.9166870117188,
       "epoch": 0.945,
-      "grad_norm": 0.6116769185552291,
-      "kl": 0.0986328125,
+      "grad_norm": 2.384828435211722,
+      "kl": 0.2900390625,
       "learning_rate": 1.0826776744855121e-07,
-      "loss": 0.0043,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.022,
+      "reward": 0.3958333432674408,
+      "reward_std": 0.29257264733314514,
+      "rewards/accuracy_reward": 0.3958333432674408,
       "step": 945
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 448.3541717529297,
+      "completion_length": 2011.2916870117188,
       "epoch": 0.946,
-      "grad_norm": 5.836343401745722,
-      "kl": 0.224853515625,
+      "grad_norm": 0.5193047463897833,
+      "kl": 0.3330078125,
       "learning_rate": 1.0797073717209013e-07,
-      "loss": 0.0643,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0135,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
       "step": 946
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 499.5833435058594,
+      "completion_length": 2041.375,
       "epoch": 0.947,
-      "grad_norm": 12.017409443755474,
-      "kl": 0.9892578125,
+      "grad_norm": 1.9164560619486717,
+      "kl": 0.39453125,
       "learning_rate": 1.0767909288270063e-07,
-      "loss": 0.2575,
-      "reward": 0.9635416865348816,
-      "reward_std": 0.1110176369547844,
-      "rewards/tag_count_reward": 0.9635416865348816,
+      "loss": 0.0158,
+      "reward": 0.20833333395421505,
+      "reward_std": 0.18523553758859634,
+      "rewards/accuracy_reward": 0.20833333395421505,
       "step": 947
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 456.9791717529297,
+      "completion_length": 2044.9583740234375,
       "epoch": 0.948,
-      "grad_norm": 2.315036725074918,
-      "kl": 0.12255859375,
+      "grad_norm": 3.498923802896285,
+      "kl": 0.3828125,
       "learning_rate": 1.0739283813397639e-07,
-      "loss": 0.0218,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0133,
+      "reward": 0.375,
+      "reward_std": 0.3083270415663719,
+      "rewards/accuracy_reward": 0.375,
       "step": 948
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 423.37501525878906,
+      "completion_length": 2045.4583740234375,
       "epoch": 0.949,
-      "grad_norm": 1.684987978433669,
-      "kl": 0.15966796875,
+      "grad_norm": 2.174961311651389,
+      "kl": 0.4375,
       "learning_rate": 1.0711197641384115e-07,
-      "loss": 0.0065,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0175,
+      "reward": 0.1458333432674408,
+      "reward_std": 0.20272701978683472,
+      "rewards/accuracy_reward": 0.1458333432674408,
       "step": 949
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 433.6666717529297,
+      "completion_length": 2011.6666870117188,
       "epoch": 0.95,
-      "grad_norm": 0.7242424715971006,
-      "kl": 0.0859375,
+      "grad_norm": 2.398594804931827,
+      "kl": 0.3720703125,
       "learning_rate": 1.068365111445064e-07,
-      "loss": 0.0039,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0149,
+      "reward": 0.0416666679084301,
+      "reward_std": 0.14433756470680237,
+      "rewards/accuracy_reward": 0.0416666679084301,
       "step": 950
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 439.4166717529297,
+      "completion_length": 2048.0,
       "epoch": 0.951,
-      "grad_norm": 1.3350386086684431,
-      "kl": 0.102294921875,
+      "grad_norm": 2.994339112920465,
+      "kl": 0.357421875,
       "learning_rate": 1.0656644568242946e-07,
-      "loss": -0.0403,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0143,
+      "reward": 0.3333333432674408,
+      "reward_std": 0.22787059843540192,
+      "rewards/accuracy_reward": 0.3333333432674408,
       "step": 951
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 463.31251525878906,
+      "completion_length": 2000.0000610351562,
       "epoch": 0.952,
-      "grad_norm": 4.247740925227692,
-      "kl": 0.31201171875,
+      "grad_norm": 2.7654370590565276,
+      "kl": 0.4140625,
       "learning_rate": 1.063017833182728e-07,
-      "loss": 0.0315,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0146,
+      "reward": 0.1875,
+      "reward_std": 0.19526028633117676,
+      "rewards/accuracy_reward": 0.1875,
       "step": 952
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 456.6458435058594,
+      "completion_length": 2018.4166870117188,
       "epoch": 0.953,
-      "grad_norm": 0.3145352011003649,
-      "kl": 0.0618896484375,
+      "grad_norm": 3.9184440517713406,
+      "kl": 0.3935546875,
       "learning_rate": 1.0604252727686379e-07,
-      "loss": 0.0025,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0086,
+      "reward": 0.3125000149011612,
+      "reward_std": 0.3139677196741104,
+      "rewards/accuracy_reward": 0.3125000149011612,
       "step": 953
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 369.1458435058594,
+      "completion_length": 2048.0,
       "epoch": 0.954,
-      "grad_norm": 0.8741994646212923,
-      "kl": 0.07708740234375,
+      "grad_norm": 3.1170785209350775,
+      "kl": 0.384765625,
       "learning_rate": 1.0578868071715544e-07,
-      "loss": 0.0039,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0154,
+      "reward": 0.0833333358168602,
+      "reward_std": 0.18523554503917694,
+      "rewards/accuracy_reward": 0.0833333358168602,
       "step": 954
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 407.85418701171875,
+      "completion_length": 2041.6250610351562,
       "epoch": 0.955,
-      "grad_norm": 1.2543903429135423,
-      "kl": 0.0997314453125,
+      "grad_norm": 5.721951876408792,
+      "kl": 0.4306640625,
       "learning_rate": 1.0554024673218806e-07,
-      "loss": 0.0056,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0157,
+      "reward": 0.3125000149011612,
+      "reward_std": 0.22604453563690186,
+      "rewards/accuracy_reward": 0.3125000149011612,
       "step": 955
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 503.62501525878906,
+      "completion_length": 2009.6458740234375,
       "epoch": 0.956,
-      "grad_norm": 3.2277603958784464,
-      "kl": 0.11083984375,
+      "grad_norm": 3.339897916363142,
+      "kl": 0.427734375,
       "learning_rate": 1.0529722834905125e-07,
-      "loss": 0.0483,
-      "reward": 0.984375,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.0001,
+      "reward": 0.3125000149011612,
+      "reward_std": 0.25182367861270905,
+      "rewards/accuracy_reward": 0.3125000149011612,
       "step": 956
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 397.3958435058594,
+      "completion_length": 1968.2291870117188,
       "epoch": 0.957,
-      "grad_norm": 1.613929883243826,
-      "kl": 0.10498046875,
+      "grad_norm": 4.010866254013547,
+      "kl": 0.3955078125,
       "learning_rate": 1.0505962852884739e-07,
-      "loss": 0.0085,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0386,
+      "reward": 0.5416666865348816,
+      "reward_std": 0.32399246096611023,
+      "rewards/accuracy_reward": 0.5416666865348816,
       "step": 957
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 414.41668701171875,
+      "completion_length": 2044.75,
       "epoch": 0.958,
-      "grad_norm": 1.923232633431884,
-      "kl": 0.09423828125,
+      "grad_norm": 4.247227383060719,
+      "kl": 0.3525390625,
       "learning_rate": 1.0482745016665526e-07,
-      "loss": 0.0127,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0148,
+      "reward": 0.4583333432674408,
+      "reward_std": 0.14433756470680237,
+      "rewards/accuracy_reward": 0.4583333432674408,
       "step": 958
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 469.8125,
+      "completion_length": 2000.166748046875,
       "epoch": 0.959,
-      "grad_norm": 2.5359963977453805,
-      "kl": 0.1854248046875,
+      "grad_norm": 1.472972711379771,
+      "kl": 0.37109375,
       "learning_rate": 1.0460069609149496e-07,
-      "loss": 0.0261,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0117,
+      "reward": 0.1041666716337204,
+      "reward_std": 0.12873217463493347,
+      "rewards/accuracy_reward": 0.1041666716337204,
       "step": 959
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 471.3958435058594,
+      "completion_length": 1954.5416870117188,
       "epoch": 0.96,
-      "grad_norm": 1.6343344818979024,
-      "kl": 0.13623046875,
+      "grad_norm": 6.229348926525174,
+      "kl": 0.396484375,
       "learning_rate": 1.0437936906629334e-07,
-      "loss": 0.0068,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": -0.0032,
+      "reward": 0.5416666865348816,
+      "reward_std": 0.3861364871263504,
+      "rewards/accuracy_reward": 0.5416666865348816,
       "step": 960
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 474.8333435058594,
+      "completion_length": 1989.3958740234375,
       "epoch": 0.961,
-      "grad_norm": 2.778614137062019,
-      "kl": 0.095458984375,
+      "grad_norm": 3.934380745153605,
+      "kl": 0.4111328125,
       "learning_rate": 1.0416347178785039e-07,
-      "loss": 0.0435,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": -0.0264,
+      "reward": 0.4583333544433117,
+      "reward_std": 0.3295731097459793,
+      "rewards/accuracy_reward": 0.4583333544433117,
       "step": 961
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 495.75001525878906,
+      "completion_length": 2045.625,
       "epoch": 0.962,
-      "grad_norm": 2.72097056019789,
-      "kl": 0.12646484375,
+      "grad_norm": 3.8313410354143294,
+      "kl": 0.427734375,
       "learning_rate": 1.0395300688680625e-07,
-      "loss": 0.0063,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0149,
+      "reward": 0.3958333432674408,
+      "reward_std": 0.29257266223430634,
+      "rewards/accuracy_reward": 0.3958333432674408,
       "step": 962
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 395.7708435058594,
+      "completion_length": 2048.0,
       "epoch": 0.963,
-      "grad_norm": 1.4073298211988017,
-      "kl": 0.18896484375,
+      "grad_norm": 4.088136999223854,
+      "kl": 0.4560546875,
       "learning_rate": 1.0374797692760933e-07,
-      "loss": -0.0007,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0182,
+      "reward": 0.375,
+      "reward_std": 0.345238521695137,
+      "rewards/accuracy_reward": 0.375,
       "step": 963
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 442.18751525878906,
+      "completion_length": 1977.4166870117188,
       "epoch": 0.964,
-      "grad_norm": 9.695014254118218,
-      "kl": 0.65673828125,
+      "grad_norm": 2.972129726625707,
+      "kl": 0.392578125,
       "learning_rate": 1.0354838440848501e-07,
-      "loss": 0.0452,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0263,
+      "reward": 0.1875,
+      "reward_std": 0.11306675523519516,
+      "rewards/accuracy_reward": 0.1875,
       "step": 964
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 453.125,
+      "completion_length": 2005.791748046875,
       "epoch": 0.965,
-      "grad_norm": 5.34938148154237,
-      "kl": 0.420166015625,
+      "grad_norm": 5.58198265819269,
+      "kl": 0.3759765625,
       "learning_rate": 1.0335423176140511e-07,
-      "loss": 0.043,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.012,
+      "reward": 0.375,
+      "reward_std": 0.20090095698833466,
+      "rewards/accuracy_reward": 0.375,
       "step": 965
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 485.85418701171875,
+      "completion_length": 2013.625,
       "epoch": 0.966,
-      "grad_norm": 4.22238362065506,
-      "kl": 0.129638671875,
+      "grad_norm": 4.88478755571173,
+      "kl": 0.55078125,
       "learning_rate": 1.0316552135205837e-07,
-      "loss": 0.0346,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": 0.0136,
+      "reward": 0.375,
+      "reward_std": 0.20090095698833466,
+      "rewards/accuracy_reward": 0.375,
       "step": 966
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 458.81251525878906,
+      "completion_length": 2014.3125,
       "epoch": 0.967,
-      "grad_norm": 0.6331126151655933,
-      "kl": 0.096435546875,
+      "grad_norm": 3.181883565590805,
+      "kl": 0.591796875,
       "learning_rate": 1.029822554798216e-07,
-      "loss": 0.0043,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0141,
+      "reward": 0.0833333358168602,
+      "reward_std": 0.19462472200393677,
+      "rewards/accuracy_reward": 0.0833333358168602,
       "step": 967
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 471.25001525878906,
+      "completion_length": 2048.0,
       "epoch": 0.968,
-      "grad_norm": 4.328218469990424,
-      "kl": 0.1728515625,
+      "grad_norm": 3.4450142495031146,
+      "kl": 0.4052734375,
       "learning_rate": 1.0280443637773163e-07,
-      "loss": 0.0843,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0162,
+      "reward": 0.4791666716337204,
+      "reward_std": 0.16948114335536957,
+      "rewards/accuracy_reward": 0.4791666716337204,
       "step": 968
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 389.62501525878906,
+      "completion_length": 2048.0,
       "epoch": 0.969,
-      "grad_norm": 0.705259059622117,
-      "kl": 0.08544921875,
+      "grad_norm": 5.344152440876086,
+      "kl": 0.5263671875,
       "learning_rate": 1.0263206621245807e-07,
-      "loss": 0.0039,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0211,
+      "reward": 0.1666666679084301,
+      "reward_std": 0.36474141478538513,
+      "rewards/accuracy_reward": 0.1666666679084301,
       "step": 969
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 389.9583435058594,
+      "completion_length": 2008.2291870117188,
       "epoch": 0.97,
-      "grad_norm": 4.207076961902249,
-      "kl": 0.16650390625,
+      "grad_norm": 3.185084963082658,
+      "kl": 0.5546875,
       "learning_rate": 1.0246514708427701e-07,
-      "loss": 0.0082,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.022,
+      "reward": 0.1041666679084301,
+      "reward_std": 0.31381870806217194,
+      "rewards/accuracy_reward": 0.1041666679084301,
       "step": 970
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 391.4583435058594,
+      "completion_length": 2041.3958740234375,
       "epoch": 0.971,
-      "grad_norm": 1.2010840276151276,
-      "kl": 0.11767578125,
+      "grad_norm": 7.481332352513289,
+      "kl": 0.498046875,
       "learning_rate": 1.0230368102704531e-07,
-      "loss": 0.0056,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0183,
+      "reward": 0.20833333395421505,
+      "reward_std": 0.26742906868457794,
+      "rewards/accuracy_reward": 0.20833333395421505,
       "step": 971
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 466.7708435058594,
+      "completion_length": 2008.9375,
       "epoch": 0.972,
-      "grad_norm": 1.8836702979327664,
-      "kl": 0.068603515625,
+      "grad_norm": 4.242146173077179,
+      "kl": 0.62890625,
       "learning_rate": 1.0214767000817596e-07,
-      "loss": 0.031,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": -0.0087,
+      "reward": 0.2708333432674408,
+      "reward_std": 0.21037911623716354,
+      "rewards/accuracy_reward": 0.2708333432674408,
       "step": 972
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 380.50001525878906,
+      "completion_length": 2048.0,
       "epoch": 0.973,
-      "grad_norm": 0.19391861126194634,
-      "kl": 0.0452880859375,
+      "grad_norm": 7.757430573014844,
+      "kl": 0.482421875,
       "learning_rate": 1.01997115928614e-07,
-      "loss": 0.0019,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0193,
+      "reward": 0.25000000558793545,
+      "reward_std": 0.14433756470680237,
+      "rewards/accuracy_reward": 0.25000000558793545,
       "step": 973
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 452.9791717529297,
+      "completion_length": 2041.4166870117188,
       "epoch": 0.974,
-      "grad_norm": 7.651648747333932,
-      "kl": 0.5869140625,
+      "grad_norm": 5.857549412120691,
+      "kl": 0.513671875,
       "learning_rate": 1.0185202062281336e-07,
-      "loss": 0.2445,
-      "reward": 0.9739583730697632,
-      "reward_std": 0.06669837608933449,
-      "rewards/tag_count_reward": 0.9739583730697632,
+      "loss": 0.0175,
+      "reward": 0.5625,
+      "reward_std": 0.2436249926686287,
+      "rewards/accuracy_reward": 0.5625,
       "step": 974
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 427.4166717529297,
+      "completion_length": 2048.0,
       "epoch": 0.975,
-      "grad_norm": 8.716776035375233,
-      "kl": 0.171875,
+      "grad_norm": 4.773417971834341,
+      "kl": 0.5185546875,
       "learning_rate": 1.017123858587145e-07,
-      "loss": 0.0764,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": 0.0207,
+      "reward": 0.25,
+      "reward_std": 0.3647414296865463,
+      "rewards/accuracy_reward": 0.25,
       "step": 975
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 405.2708435058594,
+      "completion_length": 2007.25,
       "epoch": 0.976,
-      "grad_norm": 0.8957252085487097,
-      "kl": 0.1015625,
+      "grad_norm": 8.448973471854748,
+      "kl": 0.470703125,
       "learning_rate": 1.0157821333772304e-07,
-      "loss": 0.0049,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": -0.0181,
+      "reward": 0.7916666865348816,
+      "reward_std": 0.4056393951177597,
+      "rewards/accuracy_reward": 0.7916666865348816,
       "step": 976
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 449.4166717529297,
+      "completion_length": 2011.1250610351562,
       "epoch": 0.977,
-      "grad_norm": 2.145008185762233,
-      "kl": 0.17578125,
+      "grad_norm": 5.451528487592903,
+      "kl": 0.484375,
       "learning_rate": 1.014495046946888e-07,
-      "loss": 0.0096,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.043,
+      "reward": 0.3125,
+      "reward_std": 0.3139677122235298,
+      "rewards/accuracy_reward": 0.3125,
       "step": 977
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 373.8541717529297,
+      "completion_length": 2048.0,
       "epoch": 0.978,
-      "grad_norm": 4.585846477881522,
-      "kl": 0.301025390625,
+      "grad_norm": 4.52239240528912,
+      "kl": 0.494140625,
       "learning_rate": 1.013262614978859e-07,
-      "loss": 0.0301,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0198,
+      "reward": 0.2083333358168602,
+      "reward_std": 0.34349535405635834,
+      "rewards/accuracy_reward": 0.2083333358168602,
       "step": 978
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 429.0833435058594,
+      "completion_length": 2043.75,
       "epoch": 0.979,
-      "grad_norm": 1.6114492254576476,
-      "kl": 0.0986328125,
+      "grad_norm": 4.828253388648634,
+      "kl": 0.658203125,
       "learning_rate": 1.0120848524899386e-07,
-      "loss": 0.0046,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0222,
+      "reward": 0.4375000149011612,
+      "reward_std": 0.26679351925849915,
+      "rewards/accuracy_reward": 0.4375000149011612,
       "step": 979
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 421.7708435058594,
+      "completion_length": 2012.0416870117188,
       "epoch": 0.98,
-      "grad_norm": 0.5571361949186726,
-      "kl": 0.073486328125,
+      "grad_norm": 5.5904528719875675,
+      "kl": 0.513671875,
       "learning_rate": 1.0109617738307911e-07,
-      "loss": 0.0035,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0144,
+      "reward": 0.3125000111758709,
+      "reward_std": 0.31381870806217194,
+      "rewards/accuracy_reward": 0.3125000111758709,
       "step": 980
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 397.43751525878906,
+      "completion_length": 2048.0,
       "epoch": 0.981,
-      "grad_norm": 31.643005792685624,
-      "kl": 0.489013671875,
+      "grad_norm": 8.093967265235792,
+      "kl": 0.63671875,
       "learning_rate": 1.0098933926857752e-07,
-      "loss": 0.0589,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0255,
+      "reward": 0.16666667722165585,
+      "reward_std": 0.29821331799030304,
+      "rewards/accuracy_reward": 0.16666667722165585,
       "step": 981
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 524.6250305175781,
+      "completion_length": 2016.4375,
       "epoch": 0.982,
-      "grad_norm": 3.0764806977544823,
-      "kl": 0.183349609375,
+      "grad_norm": 3.7994472447735474,
+      "kl": 0.6796875,
       "learning_rate": 1.0088797220727779e-07,
-      "loss": 0.0259,
-      "reward": 0.984375,
-      "reward_std": 0.03884884715080261,
-      "rewards/tag_count_reward": 0.984375,
+      "loss": 0.018,
+      "reward": 0.08333333395421505,
+      "reward_std": 0.18523553758859634,
+      "rewards/accuracy_reward": 0.08333333395421505,
       "step": 982
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 375.1875,
+      "completion_length": 2026.625,
       "epoch": 0.983,
-      "grad_norm": 2.6538317352580862,
-      "kl": 0.0966796875,
+      "grad_norm": 6.005690870142635,
+      "kl": 0.658203125,
       "learning_rate": 1.007920774343056e-07,
-      "loss": 0.0132,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0266,
+      "reward": 0.2500000149011612,
+      "reward_std": 0.25746434926986694,
+      "rewards/accuracy_reward": 0.2500000149011612,
       "step": 983
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 420.56251525878906,
+      "completion_length": 2009.75,
       "epoch": 0.984,
-      "grad_norm": 1.3248447884837615,
-      "kl": 0.16162109375,
+      "grad_norm": 4.309288210049117,
+      "kl": 0.4755859375,
       "learning_rate": 1.0070165611810855e-07,
-      "loss": 0.0072,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0239,
+      "reward": 0.4791666679084301,
+      "reward_std": 0.26679350435733795,
+      "rewards/accuracy_reward": 0.4791666679084301,
       "step": 984
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 413.5208435058594,
+      "completion_length": 2011.875,
       "epoch": 0.985,
-      "grad_norm": 1.8074473920007366,
-      "kl": 0.0712890625,
+      "grad_norm": 3.3491852827618316,
+      "kl": 0.775390625,
       "learning_rate": 1.0061670936044178e-07,
-      "loss": 0.0004,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0761,
+      "reward": 0.31250002048909664,
+      "reward_std": 0.16948114335536957,
+      "rewards/accuracy_reward": 0.31250002048909664,
       "step": 985
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 415.375,
+      "completion_length": 2034.25,
       "epoch": 0.986,
-      "grad_norm": 2.1831096647312727,
-      "kl": 0.15283203125,
+      "grad_norm": 3.7531852345748167,
+      "kl": 0.708984375,
       "learning_rate": 1.005372381963547e-07,
-      "loss": 0.0026,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0271,
+      "reward": 0.02083333395421505,
+      "reward_std": 0.07216878235340118,
+      "rewards/accuracy_reward": 0.02083333395421505,
       "step": 986
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 462.6041717529297,
+      "completion_length": 2048.0,
       "epoch": 0.987,
-      "grad_norm": 3.0665952226528823,
-      "kl": 0.1796875,
+      "grad_norm": 5.9330475022327684,
+      "kl": 0.44921875,
       "learning_rate": 1.0046324359417842e-07,
-      "loss": 0.0181,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0179,
+      "reward": 0.1041666679084301,
+      "reward_std": 0.25740431994199753,
+      "rewards/accuracy_reward": 0.1041666679084301,
       "step": 987
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 465.75001525878906,
+      "completion_length": 2009.6666870117188,
       "epoch": 0.988,
-      "grad_norm": 0.44700716676946056,
-      "kl": 0.07275390625,
+      "grad_norm": 5.5486659512551135,
+      "kl": 0.5693359375,
       "learning_rate": 1.0039472645551372e-07,
-      "loss": 0.0031,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0365,
+      "reward": 0.291666679084301,
+      "reward_std": 0.29193708300590515,
+      "rewards/accuracy_reward": 0.291666679084301,
       "step": 988
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 417.93751525878906,
+      "completion_length": 1980.4166870117188,
       "epoch": 0.989,
-      "grad_norm": 7.927309333392955,
-      "kl": 0.1513671875,
+      "grad_norm": 4.324549373472642,
+      "kl": 0.5419921875,
       "learning_rate": 1.0033168761522048e-07,
-      "loss": 0.1489,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": 0.0309,
+      "reward": 0.22916667722165585,
+      "reward_std": 0.29257263243198395,
+      "rewards/accuracy_reward": 0.22916667722165585,
       "step": 989
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 407.37501525878906,
+      "completion_length": 2048.0,
       "epoch": 0.99,
-      "grad_norm": 1.6469239300234044,
-      "kl": 0.191162109375,
+      "grad_norm": 7.069118392417844,
+      "kl": 0.685546875,
       "learning_rate": 1.002741278414069e-07,
-      "loss": 0.0086,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0275,
+      "reward": 0.3125000111758709,
+      "reward_std": 0.31381870806217194,
+      "rewards/accuracy_reward": 0.3125000111758709,
       "step": 990
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 448.00001525878906,
+      "completion_length": 2048.0,
       "epoch": 0.991,
-      "grad_norm": 0.21007285113485952,
-      "kl": 0.0518798828125,
+      "grad_norm": 5.400628696178434,
+      "kl": 0.923828125,
       "learning_rate": 1.0022204783542078e-07,
-      "loss": 0.002,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0369,
+      "reward": 0.25000000558793545,
+      "reward_std": 0.3083270341157913,
+      "rewards/accuracy_reward": 0.25000000558793545,
       "step": 991
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 436.8333435058594,
+      "completion_length": 2048.0,
       "epoch": 0.992,
-      "grad_norm": 2.200010988728657,
-      "kl": 0.115234375,
+      "grad_norm": 4.201918207974384,
+      "kl": 0.75390625,
       "learning_rate": 1.0017544823184055e-07,
-      "loss": 0.0112,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0302,
+      "reward": 0.14583333395421505,
+      "reward_std": 0.2983022928237915,
+      "rewards/accuracy_reward": 0.14583333395421505,
       "step": 992
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 538.6250305175781,
+      "completion_length": 2033.25,
       "epoch": 0.993,
-      "grad_norm": 4.283023167021855,
-      "kl": 0.2578125,
+      "grad_norm": 7.140839021737795,
+      "kl": 0.595703125,
       "learning_rate": 1.001343295984676e-07,
-      "loss": 0.0217,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0167,
+      "reward": 0.4166666865348816,
+      "reward_std": 0.47405362129211426,
+      "rewards/accuracy_reward": 0.4166666865348816,
       "step": 993
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 486.56251525878906,
+      "completion_length": 2032.7291870117188,
       "epoch": 0.994,
-      "grad_norm": 5.259883325340397,
-      "kl": 0.28125,
+      "grad_norm": 9.765611638762456,
+      "kl": 0.89453125,
       "learning_rate": 1.0009869243631952e-07,
-      "loss": 0.0557,
-      "reward": 0.9791666865348816,
-      "reward_std": 0.07216878235340118,
-      "rewards/tag_count_reward": 0.9791666865348816,
+      "loss": 0.0352,
+      "reward": 0.3333333432674408,
+      "reward_std": 0.4056394100189209,
+      "rewards/accuracy_reward": 0.3333333432674408,
       "step": 994
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 520.25,
+      "completion_length": 2048.0,
       "epoch": 0.995,
-      "grad_norm": 2.437461178054645,
-      "kl": 0.203369140625,
+      "grad_norm": 10.014297644087783,
+      "kl": 0.568359375,
       "learning_rate": 1.0006853717962393e-07,
-      "loss": 0.0343,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0228,
+      "reward": 0.625,
+      "reward_std": 0.4207582324743271,
+      "rewards/accuracy_reward": 0.625,
       "step": 995
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 455.0,
+      "completion_length": 2029.1875,
       "epoch": 0.996,
-      "grad_norm": 1.952234178856873,
-      "kl": 0.115234375,
+      "grad_norm": 9.627804045985279,
+      "kl": 0.67578125,
       "learning_rate": 1.000438641958131e-07,
-      "loss": -0.0091,
-      "reward": 0.9895833432674408,
-      "reward_std": 0.03608439117670059,
-      "rewards/tag_count_reward": 0.9895833432674408,
+      "loss": 0.0146,
+      "reward": 0.5208333432674408,
+      "reward_std": 0.505473405122757,
+      "rewards/accuracy_reward": 0.5208333432674408,
       "step": 996
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 427.47918701171875,
+      "completion_length": 1974.2291870117188,
       "epoch": 0.997,
-      "grad_norm": 4.669011479719766,
-      "kl": 0.1650390625,
+      "grad_norm": 8.942176643164945,
+      "kl": 0.60546875,
       "learning_rate": 1.0002467378551954e-07,
-      "loss": 0.0236,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": 0.0109,
+      "reward": 0.5625000298023224,
+      "reward_std": 0.4112800657749176,
+      "rewards/accuracy_reward": 0.5625000298023224,
       "step": 997
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 440.4791717529297,
+      "completion_length": 2034.5,
       "epoch": 0.998,
-      "grad_norm": 0.5694568503474046,
-      "kl": 0.089111328125,
+      "grad_norm": 4.813515348908535,
+      "kl": 0.857421875,
       "learning_rate": 1.0001096618257236e-07,
-      "loss": 0.0041,
-      "reward": 1.0,
-      "reward_std": 0.0,
-      "rewards/tag_count_reward": 1.0,
+      "loss": 0.0302,
+      "reward": 0.5625,
+      "reward_std": 0.37038208544254303,
+      "rewards/accuracy_reward": 0.5625,
       "step": 998
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 420.6458435058594,
+      "completion_length": 2048.0,
       "epoch": 0.999,
-      "grad_norm": 7.818586157248591,
-      "kl": 0.497802734375,
+      "grad_norm": 4.001598411947376,
+      "kl": 0.865234375,
       "learning_rate": 1.0000274155399433e-07,
-      "loss": 0.1502,
-      "reward": 0.9947916865348816,
-      "reward_std": 0.018042195588350296,
-      "rewards/tag_count_reward": 0.9947916865348816,
+      "loss": 0.0346,
+      "reward": 0.1458333358168602,
+      "reward_std": 0.29257264733314514,
+      "rewards/accuracy_reward": 0.1458333358168602,
       "step": 999
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 441.03125,
+      "completion_length": 2048.0,
       "epoch": 1.0,
-      "grad_norm": 8.490967975746036,
-      "kl": 0.4716796875,
+      "grad_norm": 8.725818068491904,
+      "kl": 0.6640625,
       "learning_rate": 1e-07,
-      "loss": 0.0229,
-      "reward": 0.9843750298023224,
-      "reward_std": 0.05412658676505089,
-      "rewards/tag_count_reward": 0.9843750298023224,
+      "loss": 0.0262,
+      "reward": 0.25,
+      "reward_std": 0.14433756470680237,
+      "rewards/accuracy_reward": 0.25,
       "step": 1000
     },
     {
       "epoch": 1.0,
       "step": 1000,
       "total_flos": 0.0,
-      "train_loss": 0.061943143279685954,
-      "train_runtime": 34538.0147,
-      "train_samples_per_second": 0.116,
-      "train_steps_per_second": 0.029
+      "train_loss": 0.09664259935633891,
+      "train_runtime": 82727.9686,
+      "train_samples_per_second": 0.048,
+      "train_steps_per_second": 0.012
     }
   ],
   "logging_steps": 1,