diff --git "a/docs/logdump.txt" "b/docs/logdump.txt"
new file mode 100644--- /dev/null
+++ "b/docs/logdump.txt"
@@ -0,0 +1,3273 @@
+Hugging Face's logo
+Hugging Face
+Models
+Datasets
+Spaces
+Buckets
+new
+Docs
+Enterprise
+Pricing
+
+
+Jobs
+:
+ InosLihka
+/
+69ed8c25d70108f37acdf744
+
+Status
+
+Canceled
+Created
+
+26/04/2026, 09:23:09
+
+Hardware
+
+a10g-large
+
+Image
+
+ghcr.io/astral-sh/uv:python3.12-bookworm
+Command
+
+bash -c 'echo $LOCAL_FILES_ENCODED | xargs -n 2 bash -c '\''echo "$1" | base64 -d > "$0"'\'' && uv run '\''train_on_hf.py'\'''
+
+Environment variables
+
+FAST_MODE=1 LOCAL_FILES_ENCODED=train_on_hf.py IyAvLy8gc2NyaXB0CiMgcmVxdWlyZXMtcHl0aG9uID0gIj49My4xMCIKIyBkZXBlbmRlbmNpZXMgPSBbCiMgICAidG9yY2giLAojICAgInRyYW5zZm9ybWVycz09NC41Ni4yIiwKIyAgICJ0cmw9PTAuMjIuMiIsCiMgICAiZGF0YXNldHMiLAojICAgInBlZnQiLAojICAgImFjY2VsZXJhdGUiLAojICAgImJpdHNhbmRieXRlcyIsCiMgICAidW5zbG90aCIsCiMgICAib3BlbmVudi1jb3JlIiwKIyAgICJmYXN0YXBpIiwKIyAgICJ1dmljb3JuIiwKIyAgICJweWRhbnRpYyIsCiMgICAibWF0cGxvdGxpYiIsCiMgICAiaHVnZ2luZ2ZhY2VfaHViIiwKIyBdCiMgLy8vCiIiIgpFbmQtdG8tZW5kIHRyYWluaW5nIGpvYiBmb3IgSEYgSm9icy4KClN1Ym1pdCBmcm9tIGxvY2FsIG1hY2hpbmUgd2l0aDoKICAgIGhmIGpvYnMgdXYgcnVuIC0tZmxhdm9yIGExMGctbGFyZ2UgLS1zZWNyZXRzIEhGX1RPS0VOIHNjcmlwdHMvdHJhaW5fb25faGYucHkKCldoYXQgaXQgZG9lcyAobm8gYmFieXNpdHRpbmcgcmVxdWlyZWQpOgogIDEuIENsb25lIHJoeXRobV9lbnYgZnJvbSBIRiBTcGFjZSAoZ2V0cyBsYXRlc3QgbWV0YS1STCBjb2RlIGZyb20gbWFpbikKICAyLiBHZW5lcmF0ZSBkYXRhc2V0IChjb250aW51b3VzIHByb2ZpbGVzLCBoaW50X2ZyYWN0aW9uPTAuMTUpCiAgMy4gVHJhaW4gUXdlbiAyLjUtM0IgKyBMb1JBIHJhbmsgOCB2aWEgR1JQTyAoMTUwMCBzdGVwcykKICA0LiBSdW4gZXZhbCBvbiBhbGwgMyBjb25kaXRpb25zIChkaXNjcmV0ZSwgaW4tZGlzdCwgT09EKQogIDUuIEdlbmVyYXRlIGFsbCA1IHBsb3RzIGZyb20gbG9nX2hpc3RvcnkKICA2LiBVcGxvYWQgdHJhaW5lZCBtb2RlbCArIHBsb3RzICsgZXZhbCBKU09OIHRvIGEgbmV3IEhGIEh1YiBtb2RlbCByZXBvCgpPdmVycmlkZSBkZWZhdWx0cyB2aWEgZW52IHZhcnM6CiAgICBNQVhfU1RFUFMsIE5VTV9FUElTT0RFUywgTE9SQV9SQU5LLCBCRVRBLCBNT0RFTF9SRVBPCgpFc3RpbWF0ZWQgY29zdCBvbiBhMTBnLWxhcmdlIGF0ICQxLjUwL2hyOiB+JDMgZm9yIDE1MDAgc3RlcHMgKH4yaCkuCiIiIgoKaW1wb3J0IGpzb24KaW1wb3J0IG9zCmltcG9ydCBzaHV0aWwKaW1wb3J0IHN1YnByb2Nlc3MKaW1wb3J0IHN5cwpmcm9tIHBhdGhsaWIgaW1wb3J0IFBhdGgKCiMgLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tCiMgQ29uZmlnIChvdmVycmlkYWJsZSB2aWEgZW52IHZhcnMpCiMgLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tClJFUE9fVVJMID0gb3MuZW52aXJvbi5nZXQoIlJFUE9fVVJMIiwgImh0dHBzOi8vaHVnZ2luZ2ZhY2UuY28vc3BhY2VzL0lub3NMaWhrYS9yaHl0aG1fZW52IikKV09SS19ESVIgPSAiL3RtcC9yaHl0aG1fZW52IgpPVVRQVVRfRElSID0gIi90bXAvcmh5dGhtX2Vudi9vdXRwdXRzL3JoeXRobWVudl9tZXRhX3RyYWluZWQiClBMT1RTX0RJUiA9ICIvdG1wL3JoeXRobV9lbnYvcGxvdHMiCgojIEZBU1RfTU9ERSBwcmVzZXQ6IH4xMC0xNSBtaW4gaXRlcmF0aW9uIG9uIEExMDAgbGFyZ2UuCiMgVXNlIGZvciBoeXBlcnBhcmFtZXRlciBzd2VlcHMgYW5kIHBpcGVsaW5lIGRlYnVnZ2luZy4KRkFTVF9NT0RFID0gb3MuZW52aXJvbi5nZXQoIkZBU1RfTU9ERSIsICIwIikgPT0gIjEiCgppZiBGQVNUX01PREU6CiAgICAjIEl0ZXIgMyBwcmVzZXQ6IDgwMCBzdGVwcyArIDggZ2VuZXJhdGlvbnMgKyBMb1JBIDE2IHRvIGVzY2FwZSBtb2RlIGNvbGxhcHNlIGZvciByZWFsCiAgICBERUZBVUxUUyA9IGRpY3QoTUFYX1NURVBTPTgwMCwgTlVNX0VQSVNPREVTPTIwMCwgTUFYX1NBTVBMRVM9MjAwMCwKICAgICAgICAgICAgICAgICAgICBOVU1fR0VORVJBVElPTlM9OCwgTE9SQV9SQU5LPTE2LCBCRVRBPTAuMDQsCiAgICAgICAgICAgICAgICAgICAgTEVBUk5JTkdfUkFURT01ZS01LCBFVkFMX0VQSVNPREVTPTIpCmVsc2U6CiAgICBERUZBVUxUUyA9IGRpY3QoTUFYX1NURVBTPTIwMDAsIE5VTV9FUElTT0RFUz00MDAsIE1BWF9TQU1QTEVTPTQwMDAsCiAgICAgICAgICAgICAgICAgICAgTlVNX0dFTkVSQVRJT05TPTgsIExPUkFfUkFOSz0xNiwgQkVUQT0wLjA0LAogICAgICAgICAgICAgICAgICAgIExFQVJOSU5HX1JBVEU9NWUtNSwgRVZBTF9FUElTT0RFUz01KQoKTUFYX1NURVBTID0gaW50KG9zLmVudmlyb24uZ2V0KCJNQVhfU1RFUFMiLCBzdHIoREVGQVVMVFNbIk1BWF9TVEVQUyJdKSkpCk5VTV9FUElTT0RFUyA9IGludChvcy5lbnZpcm9uLmdldCgiTlVNX0VQSVNPREVTIiwgc3RyKERFRkFVTFRTWyJOVU1fRVBJU09ERVMiXSkpKQpNQVhfU0FNUExFUyA9IGludChvcy5lbnZpcm9uLmdldCgiTUFYX1NBTVBMRVMiLCBzdHIoREVGQVVMVFNbIk1BWF9TQU1QTEVTIl0pKSkKTlVNX0dFTkVSQVRJT05TID0gaW50KG9zLmVudmlyb24uZ2V0KCJOVU1fR0VORVJBVElPTlMiLCBzdHIoREVGQVVMVFNbIk5VTV9HRU5FUkFUSU9OUyJdKSkpCkxPUkFfUkFOSyA9IGludChvcy5lbnZpcm9uLmdldCgiTE9SQV9SQU5LIiwgc3RyKERFRkFVTFRTWyJMT1JBX1JBTksiXSkpKQpCRVRBID0gZmxvYXQob3MuZW52aXJvbi5nZXQoIkJFVEEiLCBzdHIoREVGQVVMVFNbIkJFVEEiXSkpKQpMRUFSTklOR19SQVRFID0gZmxvYXQob3MuZW52aXJvbi5nZXQoIkxFQVJOSU5HX1JBVEUiLCBzdHIoREVGQVVMVFNbIkxFQVJOSU5HX1JBVEUiXSkpKQpFVkFMX0VQSVNPREVTID0gaW50KG9zLmVudmlyb24uZ2V0KCJFVkFMX0VQSVNPREVTIiwgc3RyKERFRkFVTFRTWyJFVkFMX0VQSVNPREVTIl0pKSkKCiMgRWFjaCBpdGVyYXRpb24gdXBsb2FkcyB0byBhIHVuaXF1ZSByZXBvIGlmIE1PREVMX1JFUE9fU1VGRklYIGlzIHNldApTVUZGSVggPSBvcy5lbnZpcm9uLmdldCgiTU9ERUxfUkVQT19TVUZGSVgiLCAiIikKREVGQVVMVF9SRVBPID0gIklub3NMaWhrYS9yaHl0aG0tZW52LW1ldGEtdHJhaW5lZCIgKyAoZiIte1NVRkZJWH0iIGlmIFNVRkZJWCBlbHNlICIiKQpNT0RFTF9SRVBPID0gb3MuZW52aXJvbi5nZXQoIk1PREVMX1JFUE8iLCBERUZBVUxUX1JFUE8pCgpwcmludChmIj09PSBSdW4gY29uZmlnID09PSIpCnByaW50KGYiICBGQVNUX01PREU6IHtGQVNUX01PREV9IikKcHJpbnQoZiIgIE1BWF9TVEVQUz17TUFYX1NURVBTfSwgTlVNX0VQSVNPREVTPXtOVU1fRVBJU09ERVN9LCBNQVhfU0FNUExFUz17TUFYX1NBTVBMRVN9IikKcHJpbnQoZiIgIE5VTV9HRU5FUkFUSU9OUz17TlVNX0dFTkVSQVRJT05TfSwgTE9SQV9SQU5LPXtMT1JBX1JBTkt9LCBCRVRBPXtCRVRBfSIpCnByaW50KGYiICBMRUFSTklOR19SQVRFPXtMRUFSTklOR19SQVRFfSwgRVZBTF9FUElTT0RFUz17RVZBTF9FUElTT0RFU30iKQpwcmludChmIiAgTU9ERUxfUkVQTz17TU9ERUxfUkVQT30iKQpwcmludCgpCgoKZGVmIHJ1bihjbWQ6IGxpc3Rbc3RyXSwgKiprdyk6CiAgICAiIiJSdW4gc3VicHJvY2VzcyB3aXRoIGxvZ2dpbmcuIiIiCiAgICBwcmludChmIlxuPj4+IHsnICcuam9pbihjbWQpIGlmIGlzaW5zdGFuY2UoY21kLCBsaXN0KSBlbHNlIGNtZH0iLCBmbHVzaD1UcnVlKQogICAgc3VicHJvY2Vzcy5ydW4oY21kLCBjaGVjaz1UcnVlLCAqKmt3KQoKCmRlZiBtYWluKCk6CiAgICAjIC0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLQogICAgIyAxLiBDbG9uZSB0aGUgcmh5dGhtX2VudiByZXBvCiAgICAjIC0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLQogICAgaWYgUGF0aChXT1JLX0RJUikuZXhpc3RzKCk6CiAgICAgICAgc2h1dGlsLnJtdHJlZShXT1JLX0RJUikKICAgIHJ1bihbImdpdCIsICJjbG9uZSIsIFJFUE9fVVJMLCBXT1JLX0RJUl0pCiAgICBvcy5jaGRpcihXT1JLX0RJUikKICAgIHN5cy5wYXRoLmluc2VydCgwLCBXT1JLX0RJUikKICAgIHN5cy5wYXRoLmluc2VydCgwLCBvcy5wYXRoLmpvaW4oV09SS19ESVIsICJ0cmFpbmluZyIpKQoKICAgICMgVmVyaWZ5IG1ldGEtUkwgY29kZSBpcyBwcmVzZW50CiAgICBkYXRhc2V0X3B5ID0gUGF0aCgidHJhaW5pbmcvZGF0YXNldC5weSIpLnJlYWRfdGV4dCgpCiAgICBhc3NlcnQgInByb2ZpbGVfbW9kZSIgaW4gZGF0YXNldF9weSwgIkNsb25lZCByZXBvIGRvZXNuJ3QgaGF2ZSBtZXRhLVJMIGNvZGUiCiAgICBwcmludCgiT0s6IG1ldGEtUkwgY29kZSBwcmVzZW50IGluIGNsb25lZCByZXBvIikKCiAgICAjIC0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLQogICAgIyAyLiBUcmFpbgogICAgIyAtLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0KICAgIHRyYWluX2FyZ3MgPSBbCiAgICAgICAgInB5dGhvbiIsICJ0cmFpbmluZy90cmFpbi5weSIsCiAgICAgICAgIi0tbWF4X3N0ZXBzIiwgc3RyKE1BWF9TVEVQUyksCiAgICAgICAgIi0tbnVtX2VwaXNvZGVzIiwgc3RyKE5VTV9FUElTT0RFUyksCiAgICAgICAgIi0tbWF4X3NhbXBsZXMiLCBzdHIoTUFYX1NBTVBMRVMpLAogICAgICAgICItLW51bV9nZW5lcmF0aW9ucyIsIHN0cihOVU1fR0VORVJBVElPTlMpLAogICAgICAgICItLWxvcmFfcmFuayIsIHN0cihMT1JBX1JBTkspLAogICAgICAgICItLWJldGEiLCBzdHIoQkVUQSksCiAgICAgICAgIi0tbGVhcm5pbmdfcmF0ZSIsIHN0cihMRUFSTklOR19SQVRFKSwKICAgICAgICAiLS1vdXRwdXRfZGlyIiwgT1VUUFVUX0RJUiwKICAgIF0KICAgIHJ1bih0cmFpbl9hcmdzKQoKICAgICMgLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tCiAgICAjIDMuIEV2YWwgKDMgY29uZGl0aW9uczogZGlzY3JldGUtMyAvIGluLWRpc3QgLyBPT0QpCiAgICAjIC0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLQogICAgZXZhbF9hcmdzID0gWwogICAgICAgICJweXRob24iLCAidHJhaW5pbmcvaW5mZXJlbmNlX2V2YWwucHkiLAogICAgICAgICItLW1vZGVsX3BhdGgiLCBPVVRQVVRfRElSLAogICAgICAgICItLW51bV9lcGlzb2RlcyIsIHN0cihFVkFMX0VQSVNPREVTKSwKICAgICAgICAiLS1vdXRwdXRfZmlsZSIsICJldmFsX3Jlc3VsdHMuanNvbiIsCiAgICBdCiAgICBydW4oZXZhbF9hcmdzKQoKICAgICMgLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tCiAgICAjIDQuIEdlbmVyYXRlIHBsb3RzIGZyb20gc2F2ZWQgbG9nX2hpc3RvcnkKICAgICMgLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tCiAgICBQYXRoKFBMT1RTX0RJUikubWtkaXIoZXhpc3Rfb2s9VHJ1ZSkKICAgIGxvZ19wYXRoID0gb3MucGF0aC5qb2luKE9VVFBVVF9ESVIsICJsb2dfaGlzdG9yeS5qc29uIikKICAgIGlmIFBhdGgobG9nX3BhdGgpLmV4aXN0cygpOgogICAgICAgIHJ1bihbInB5dGhvbiIsICJzY3JpcHRzL3Bsb3RfZnJvbV9sb2cucHkiLCAiLS1sb2ciLCBsb2dfcGF0aCwgIi0tb3V0IiwgUExPVFNfRElSXSkKICAgIGVsc2U6CiAgICAgICAgcHJpbnQoZiJXQVJOSU5HOiBsb2dfaGlzdG9yeS5qc29uIG5vdCBmb3VuZCBhdCB7bG9nX3BhdGh9IikKCiAgICAjIC0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLQogICAgIyA1LiBVcGxvYWQgZXZlcnl0aGluZyB0byBIRiBIdWIKICAgICMgLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tCiAgICB0b2tlbiA9IG9zLmVudmlyb24uZ2V0KCJIRl9UT0tFTiIpCiAgICBpZiBub3QgdG9rZW46CiAgICAgICAgcHJpbnQoIldBUk5JTkc6IEhGX1RPS0VOIG5vdCBzZXQsIHNraXBwaW5nIHVwbG9hZCIpCiAgICAgICAgcHJpbnQoZiJPdXRwdXRzIGluOiB7T1VUUFVUX0RJUn0iKQogICAgICAgIHJldHVybgoKICAgIGZyb20gaHVnZ2luZ2ZhY2VfaHViIGltcG9ydCBIZkFwaSwgbG9naW4KICAgIGxvZ2luKHRva2VuPXRva2VuKQogICAgYXBpID0gSGZBcGkoKQogICAgYXBpLmNyZWF0ZV9yZXBvKE1PREVMX1JFUE8sIGV4aXN0X29rPVRydWUsIHJlcG9fdHlwZT0ibW9kZWwiKQoKICAgICMgVXBsb2FkIHRyYWluZWQgbW9kZWwgKyBjb25maWcgKyBsb2dfaGlzdG9yeQogICAgYXBpLnVwbG9hZF9mb2xkZXIoCiAgICAgICAgZm9sZGVyX3BhdGg9T1VUUFVUX0RJUiwKICAgICAgICByZXBvX2lkPU1PREVMX1JFUE8sCiAgICAgICAgcmVwb190eXBlPSJtb2RlbCIsCiAgICAgICAgY29tbWl0X21lc3NhZ2U9ZiJUcmFpbmVkIHtNQVhfU1RFUFN9LXN0ZXAgR1JQTyBtZXRhLVJMIGFnZW50IiwKICAgICkKCiAgICAjIFVwbG9hZCBldmFsIEpTT04KICAgIGFwaS51cGxvYWRfZmlsZSgKICAgICAgICBwYXRoX29yX2ZpbGVvYmo9ImV2YWxfcmVzdWx0cy5qc29uIiwKICAgICAgICBwYXRoX2luX3JlcG89ImV2YWxfcmVzdWx0cy5qc29uIiwKICAgICAgICByZXBvX2lkPU1PREVMX1JFUE8sCiAgICAgICAgcmVwb190eXBlPSJtb2RlbCIsCiAgICApCgogICAgIyBVcGxvYWQgcGxvdHMgaWYgZ2VuZXJhdGVkCiAgICBpZiBQYXRoKFBMT1RTX0RJUikuZXhpc3RzKCkgYW5kIGFueShQYXRoKFBMT1RTX0RJUikuaXRlcmRpcigpKToKICAgICAgICBhcGkudXBsb2FkX2ZvbGRlcigKICAgICAgICAgICAgZm9sZGVyX3BhdGg9UExPVFNfRElSLAogICAgICAgICAgICBwYXRoX2luX3JlcG89InBsb3RzIiwKICAgICAgICAgICAgcmVwb19pZD1NT0RFTF9SRVBPLAogICAgICAgICAgICByZXBvX3R5cGU9Im1vZGVsIiwKICAgICAgICApCgogICAgcHJpbnQoKQogICAgcHJpbnQoIj0iICogNjApCiAgICBwcmludCgiRE9ORSIpCiAgICBwcmludChmIiAgVHJhaW5lZCBtb2RlbDogaHR0cHM6Ly9odWdnaW5nZmFjZS5jby97TU9ERUxfUkVQT30iKQogICAgcHJpbnQoZiIgIEV2YWwgSlNPTjogICAgIGh0dHBzOi8vaHVnZ2luZ2ZhY2UuY28ve01PREVMX1JFUE99L2Jsb2IvbWFpbi9ldmFsX3Jlc3VsdHMuanNvbiIpCiAgICBwcmludChmIiAgUGxvdHM6ICAgICAgICAgaHR0cHM6Ly9odWdnaW5nZmFjZS5jby97TU9ERUxfUkVQT30vdHJlZS9tYWluL3Bsb3RzIikKICAgIHByaW50KCI9IiAqIDYwKQoKCmlmIF9fbmFtZV9fID09ICJfX21haW5fXyI6CiAgICBtYWluKCkK MODEL_REPO_SUFFIX=iter4
+Secrets
+
+HF_TOKEN
+Logs
+===== Job started at 2026-04-26 03:53:10 =====
+
+Downloading hf-xet (4.0MiB)
+
+Downloading pydantic-core (2.0MiB)
+
+Downloading kiwisolver (1.4MiB)
+
+Downloading pyarrow (46.6MiB)
+
+Downloading transformers (11.1MiB)
+
+Downloading hf-transfer (3.4MiB)
+
+Downloading nvidia-cufile-cu12 (1.1MiB)
+
+Downloading cryptography (4.5MiB)
+
+Downloading networkx (2.0MiB)
+
+Downloading torchvision (7.7MiB)
+
+Downloading nvidia-nvjitlink-cu12 (37.4MiB)
+
+Downloading pillow (6.8MiB)
+
+Downloading pygments (1.2MiB)
+
+Downloading unsloth (63.9MiB)
+
+Downloading openai (1.1MiB)
+
+Downloading sympy (6.0MiB)
+
+Downloading brotli (1.4MiB)
+
+Downloading fonttools (4.8MiB)
+
+Downloading triton (179.5MiB)
+
+Downloading numpy (15.9MiB)
+
+Downloading nvidia-cudnn-cu12 (674.0MiB)
+
+Downloading aiohttp (1.7MiB)
+
+Downloading nvidia-curand-cu12 (60.7MiB)
+
+Downloading nvidia-cusparselt-cu12 (273.9MiB)
+
+Downloading nvidia-cuda-nvrtc-cu12 (84.0MiB)
+
+Downloading nvidia-cuda-cupti-cu12 (9.8MiB)
+
+Downloading nvidia-nccl-cu12 (307.4MiB)
+
+Downloading tokenizers (3.1MiB)
+
+Downloading matplotlib (8.4MiB)
+
+Downloading nvidia-nvshmem-cu12 (132.7MiB)
+
+Downloading pandas (10.4MiB)
+
+Downloading xformers (3.1MiB)
+
+Downloading beartype (1.3MiB)
+
+Downloading torch (873.2MiB)
+
+Downloading nvidia-cufft-cu12 (184.2MiB)
+
+Downloading nvidia-cusparse-cu12 (274.9MiB)
+
+Downloading nvidia-cusolver-cu12 (255.1MiB)
+
+Downloading gradio (18.8MiB)
+
+Downloading bitsandbytes (57.8MiB)
+
+Downloading nvidia-cublas-cu12 (566.8MiB)
+
+Downloading diffusers (4.8MiB)
+
+Downloading torchao (3.1MiB)
+
+Downloading cuda-bindings (11.6MiB)
+
+Downloading sentencepiece (1.3MiB)
+
+ Downloaded nvidia-cufile-cu12
+
+ Downloaded pygments
+
+ Downloaded beartype
+
+ Downloaded sentencepiece
+
+ Downloaded brotli
+
+ Downloaded kiwisolver
+
+ Downloaded aiohttp
+
+ Downloaded pydantic-core
+
+ Downloaded networkx
+
+ Downloaded tokenizers
+
+ Downloaded openai
+
+ Downloaded xformers
+
+ Downloaded hf-transfer
+
+ Downloaded hf-xet
+
+ Downloaded cryptography
+
+ Downloaded torchao
+
+ Downloaded fonttools
+
+ Downloaded diffusers
+
+ Downloaded pillow
+
+ Downloaded torchvision
+
+ Downloaded sympy
+
+ Downloaded matplotlib
+
+ Downloaded nvidia-cuda-cupti-cu12
+
+ Downloaded cuda-bindings
+
+ Downloaded transformers
+
+ Downloaded numpy
+
+ Downloaded pandas
+
+ Downloaded gradio
+
+ Downloaded nvidia-nvjitlink-cu12
+
+ Downloaded bitsandbytes
+
+ Downloaded nvidia-curand-cu12
+
+ Downloaded pyarrow
+
+ Downloaded nvidia-cuda-nvrtc-cu12
+
+ Downloaded unsloth
+
+ Downloaded nvidia-nvshmem-cu12
+
+ Downloaded nvidia-cufft-cu12
+
+ Downloaded triton
+
+ Downloaded nvidia-cusolver-cu12
+
+ Downloaded nvidia-cusparselt-cu12
+
+ Downloaded nvidia-cusparse-cu12
+
+ Downloaded nvidia-nccl-cu12
+
+ Downloaded nvidia-cublas-cu12
+
+ Downloaded nvidia-cudnn-cu12
+
+ Downloaded torch
+
+Installed 172 packages in 531ms
+
+=== Run config ===
+
+  FAST_MODE: True
+
+  MAX_STEPS=800, NUM_EPISODES=200, MAX_SAMPLES=2000
+
+  NUM_GENERATIONS=8, LORA_RANK=16, BETA=0.04
+
+  LEARNING_RATE=5e-05, EVAL_EPISODES=2
+
+  MODEL_REPO=InosLihka/rhythm-env-meta-trained-iter4
+
+
+
+>>> git clone https://huggingface.co/spaces/InosLihka/rhythm_env /tmp/rhythm_env
+
+Cloning into '/tmp/rhythm_env'...
+
+OK: meta-RL code present in cloned repo
+
+
+>>> python training/train.py --max_steps 800 --num_episodes 200 --max_samples 2000 --num_generations 8 --lora_rank 16 --beta 0.04 --learning_rate 5e-05 --output_dir /tmp/rhythm_env/outputs/rhythmenv_meta_trained
+
+============================================================
+
+Step 1: Generating training dataset (continuous profiles)
+
+============================================================
+
+Generated 2000 samples from 72 episodes (0 with profile hint, 2000 without)
+
+Dataset size: 2000
+
+
+============================================================
+
+Step 2: Loading model unsloth/Qwen2.5-3B-Instruct
+
+============================================================
+
+🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.
+
+🦥 Unsloth Zoo will now patch everything to make training faster!
+
+==((====))==  Unsloth 2026.4.8: Fast Qwen2 patching. Transformers: 4.56.2.
+
+   \\   /|    NVIDIA A10G. Num GPUs = 1. Max memory: 22.301 GB. Platform: Linux.
+
+O^O/ \_/ \    Torch: 2.10.0+cu128. CUDA: 8.6. CUDA Toolkit: 12.8. Triton: 3.6.0
+
+\        /    Bfloat16 = TRUE. FA [Xformers = 0.0.35. FA2 = False]
+
+ "-____-"     Free license: http://github.com/unslothai/unsloth
+
+Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!
+
+
+
+model.safetensors:   0%|          | 0.00/2.36G [00:00<?, ?B/s]
+
+
+model.safetensors:   3%|▎         | 68.0M/2.36G [00:01<00:36, 63.0MB/s]
+
+
+model.safetensors:  93%|█████████▎| 2.18G/2.36G [00:02<00:00, 1.23GB/s]
+model.safetensors: 100%|██████████| 2.36G/2.36G [00:02<00:00, 879MB/s] 
+
+
+
+generation_config.json:   0%|          | 0.00/271 [00:00<?, ?B/s]
+generation_config.json: 100%|██████████| 271/271 [00:00<00:00, 3.06MB/s]
+
+
+
+tokenizer_config.json:   0%|          | 0.00/7.36k [00:00<?, ?B/s]
+tokenizer_config.json: 100%|██████████| 7.36k/7.36k [00:00<00:00, 55.0MB/s]
+
+
+
+vocab.json:   0%|          | 0.00/2.78M [00:00<?, ?B/s]
+vocab.json: 100%|██████████| 2.78M/2.78M [00:00<00:00, 38.8MB/s]
+
+
+
+merges.txt:   0%|          | 0.00/1.67M [00:00<?, ?B/s]
+merges.txt: 100%|██████████| 1.67M/1.67M [00:00<00:00, 22.6MB/s]
+
+
+
+added_tokens.json:   0%|          | 0.00/605 [00:00<?, ?B/s]
+added_tokens.json: 100%|██████████| 605/605 [00:00<00:00, 8.49MB/s]
+
+
+
+special_tokens_map.json:   0%|          | 0.00/614 [00:00<?, ?B/s]
+special_tokens_map.json: 100%|██████████| 614/614 [00:00<00:00, 8.67MB/s]
+
+
+
+tokenizer.json:   0%|          | 0.00/11.4M [00:00<?, ?B/s]
+tokenizer.json: 100%|██████████| 11.4M/11.4M [00:00<00:00, 53.6MB/s]
+
+unsloth/qwen2.5-3b-instruct-unsloth-bnb-4bit does not have a padding token! Will use pad_token = <|PAD_TOKEN|>.
+
+Unsloth 2026.4.8 patched 36 layers with 36 QKV layers, 36 O layers and 36 MLP layers.
+
+LoRA rank: 16, alpha: 32
+
+
+============================================================
+
+Step 3: Setting up reward functions
+
+============================================================
+
+Using: format_valid + action_legal + env_reward + belief_accuracy
+
+
+============================================================
+
+Step 4: Configuring GRPO trainer
+
+============================================================
+
+Unsloth: We now expect `per_device_train_batch_size` * `gradient_accumulation_steps` * `world_size` to be a multiple of `num_generations`.
+
+We will change the batch size of 1 to the `num_generations` of 8
+
+Using GRPOConfig with reward_weights=[0.05, 0.05, 1.5, 3.0]
+
+max_steps=800, num_generations=8, lr=5e-05, beta=0.04
+
+max_prompt_length=600, max_completion_length=32
+
+hint_fraction=0.0 (curriculum warmup)
+
+
+============================================================
+
+Step 5: Starting GRPO training
+
+============================================================
+
+==((====))==  Unsloth - 2x faster free finetuning | Num GPUs used = 1
+
+   \\   /|    Num examples = 2,000 | Num Epochs = 2 | Total steps = 800
+
+O^O/ \_/ \    Batch size per device = 8 | Gradient accumulation steps = 4
+
+\        /    Data Parallel GPUs = 1 | Total batch size (8 x 4 x 1) = 32
+
+ "-____-"     Trainable parameters = 29,933,568 of 3,115,872,256 (0.96% trained)
+
+
+Unsloth: Will smartly offload gradients to save VRAM!
+
+
+  0%|          | 0/800 [00:00<?, ?it/s]
+
+{'loss': -0.0, 'grad_norm': 36.08258819580078, 'learning_rate': 0.0, 'num_tokens': 19974.0, 'completions/mean_length': 24.1875, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.375, 'completions/mean_terminated_length': 19.5, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': -1.125, 'rewards/format_valid/std': 1.2313119173049927, 'rewards/action_legal/mean': -0.65625, 'rewards/action_legal/std': 0.4825586974620819, 'rewards/env_reward/mean': -1.874782919883728, 'rewards/env_reward/std': 1.6368685960769653, 'rewards/belief_accuracy/mean': -0.09578604996204376, 'rewards/belief_accuracy/std': 0.023837711662054062, 'reward': -3.1885948181152344, 'reward_std': 2.168053150177002, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': -8.149072527885437e-10, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.0}
+
+
+  0%|          | 1/800 [03:18<43:57:08, 198.03s/it]
+
+
+                                                   
+
+
+
+  0%|          | 1/800 [03:18<43:57:08, 198.03s/it]
+
+
+  0%|          | 2/800 [03:33<20:08:56, 90.90s/it] 
+
+
+                                                  
+
+
+{'loss': -0.0, 'grad_norm': 35.3685188293457, 'learning_rate': 6.25e-07, 'num_tokens': 39898.0, 'completions/mean_length': 22.625, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.40625, 'completions/mean_terminated_length': 16.210525512695312, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 28.0, 'rewards/format_valid/mean': -1.28125, 'rewards/format_valid/std': 1.1704527139663696, 'rewards/action_legal/mean': -0.71875, 'rewards/action_legal/std': 0.45680341124534607, 'rewards/env_reward/mean': -2.105933666229248, 'rewards/env_reward/std': 1.464045524597168, 'rewards/belief_accuracy/mean': -0.09861111640930176, 'rewards/belief_accuracy/std': 0.007856741547584534, 'reward': -3.5547335147857666, 'reward_std': 1.6820682287216187, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': -6.402842700481415e-10, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.0}
+
+
+  0%|          | 2/800 [03:33<20:08:56, 90.90s/it]
+
+{'loss': 0.0004, 'grad_norm': 22.421438217163086, 'learning_rate': 1.25e-06, 'num_tokens': 59913.0, 'completions/mean_length': 25.46875, 'completions/min_length': 10.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.53125, 'completions/mean_terminated_length': 18.066667556762695, 'completions/min_terminated_length': 10.0, 'completions/max_terminated_length': 27.0, 'rewards/format_valid/mean': -1.59375, 'rewards/format_valid/std': 0.9625025987625122, 'rewards/action_legal/mean': -0.84375, 'rewards/action_legal/std': 0.3689020276069641, 'rewards/env_reward/mean': -2.5606305599212646, 'rewards/env_reward/std': 1.0509716272354126, 'rewards/belief_accuracy/mean': -0.09691542387008667, 'rewards/belief_accuracy/std': 0.017448989674448967, 'reward': -4.253566741943359, 'reward_std': 1.3898957967758179, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 0.010594317922368646, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.01}
+
+
+  0%|          | 3/800 [03:49<12:32:41, 56.66s/it]
+
+
+                                                  
+
+
+
+  0%|          | 3/800 [03:49<12:32:41, 56.66s/it]
+
+
+  0%|          | 4/800 [04:05<8:58:19, 40.58s/it] 
+
+
+                                                 
+
+
+{'loss': 0.0005, 'grad_norm': 71.98435974121094, 'learning_rate': 1.875e-06, 'num_tokens': 79962.0, 'completions/mean_length': 26.53125, 'completions/min_length': 12.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.5625, 'completions/mean_terminated_length': 19.5, 'completions/min_terminated_length': 12.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': -1.609375, 'rewards/format_valid/std': 0.9222550392150879, 'rewards/action_legal/mean': -0.84375, 'rewards/action_legal/std': 0.3689020276069641, 'rewards/env_reward/mean': -2.5793209075927734, 'rewards/env_reward/std': 1.0184788703918457, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': -4.291637420654297, 'reward_std': 1.660891056060791, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.012262104079127312, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.01}
+
+
+  0%|          | 4/800 [04:05<8:58:19, 40.58s/it]
+
+{'loss': 0.002, 'grad_norm': 93.35130310058594, 'learning_rate': 2.5e-06, 'num_tokens': 99963.0, 'completions/mean_length': 25.03125, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.53125, 'completions/mean_terminated_length': 17.13333511352539, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': -1.6875, 'rewards/format_valid/std': 0.84002685546875, 'rewards/action_legal/mean': -0.875, 'rewards/action_legal/std': 0.33601075410842896, 'rewards/env_reward/mean': -2.638253927230835, 'rewards/env_reward/std': 0.9890956878662109, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': -4.385505676269531, 'reward_std': 1.3020613193511963, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 0.05075692548416555, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.01}
+
+
+  1%|          | 5/800 [04:21<7:00:03, 31.70s/it]
+
+
+                                                 
+
+
+
+  1%|          | 5/800 [04:21<7:00:03, 31.70s/it]
+
+
+  1%|          | 6/800 [06:42<15:12:04, 68.92s/it]
+
+
+                                                  
+
+
+{'loss': 0.0759, 'grad_norm': 46910.00390625, 'learning_rate': 3.125e-06, 'num_tokens': 119387.0, 'completions/mean_length': 21.0, 'completions/min_length': 9.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.4375, 'completions/mean_terminated_length': 12.44444465637207, 'completions/min_terminated_length': 9.0, 'completions/max_terminated_length': 27.0, 'rewards/format_valid/mean': -0.265625, 'rewards/format_valid/std': 1.4645508527755737, 'rewards/action_legal/mean': -0.40625, 'rewards/action_legal/std': 0.49899089336395264, 'rewards/env_reward/mean': -0.8366085886955261, 'rewards/env_reward/std': 1.8311320543289185, 'rewards/belief_accuracy/mean': -0.09299995750188828, 'rewards/belief_accuracy/std': 0.07868243008852005, 'reward': -1.5675065517425537, 'reward_std': 1.2884094715118408, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.8970044270390645, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.01}
+
+
+  1%|          | 6/800 [06:42<15:12:04, 68.92s/it]
+
+{'loss': 0.0025, 'grad_norm': 76.06964111328125, 'learning_rate': 3.75e-06, 'num_tokens': 139328.0, 'completions/mean_length': 23.15625, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.40625, 'completions/mean_terminated_length': 17.105262756347656, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 29.0, 'rewards/format_valid/mean': -1.1875, 'rewards/format_valid/std': 1.3243988752365112, 'rewards/action_legal/mean': -0.71875, 'rewards/action_legal/std': 0.45680341124534607, 'rewards/env_reward/mean': -2.1376194953918457, 'rewards/env_reward/std': 1.4015222787857056, 'rewards/belief_accuracy/mean': -0.07277165353298187, 'rewards/belief_accuracy/std': 0.05686986818909645, 'reward': -3.52005672454834, 'reward_std': 1.3071869611740112, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 0.06128474697470665, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.01}
+
+
+  1%|          | 7/800 [06:58<11:21:57, 51.60s/it]
+
+
+                                                  
+
+
+
+  1%|          | 7/800 [06:58<11:21:57, 51.60s/it]
+
+
+  1%|          | 8/800 [07:14<8:51:13, 40.24s/it] 
+
+
+                                                 
+
+
+{'loss': 0.0005, 'grad_norm': 17.92104148864746, 'learning_rate': 4.375e-06, 'num_tokens': 159222.0, 'completions/mean_length': 21.6875, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.46875, 'completions/mean_terminated_length': 12.588234901428223, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 23.0, 'rewards/format_valid/mean': -0.859375, 'rewards/format_valid/std': 1.4098408222198486, 'rewards/action_legal/mean': -0.59375, 'rewards/action_legal/std': 0.49899089336395264, 'rewards/env_reward/mean': -1.5010709762573242, 'rewards/env_reward/std': 1.8662097454071045, 'rewards/belief_accuracy/mean': -0.07025672495365143, 'rewards/belief_accuracy/std': 0.060049813240766525, 'reward': -2.5350329875946045, 'reward_std': 1.7440121173858643, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.01138769625686109, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.02}
+
+
+  1%|          | 8/800 [07:14<8:51:13, 40.24s/it]
+
+
+  1%|          | 9/800 [07:30<7:10:24, 32.65s/it]
+
+
+                                                 
+
+
+{'loss': 0.0008, 'grad_norm': 8.485434532165527, 'learning_rate': 5e-06, 'num_tokens': 179288.0, 'completions/mean_length': 27.0625, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.5625, 'completions/mean_terminated_length': 20.71428680419922, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': -1.53125, 'rewards/format_valid/std': 0.9913944005966187, 'rewards/action_legal/mean': -0.8125, 'rewards/action_legal/std': 0.3965577781200409, 'rewards/env_reward/mean': -2.406738042831421, 'rewards/env_reward/std': 1.2550357580184937, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': -4.027294158935547, 'reward_std': 1.2735371589660645, 'frac_reward_zero_std': 0.5, 'completion_length': 32.0, 'kl': 0.020753059769049287, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.02}
+
+
+  1%|          | 9/800 [07:30<7:10:24, 32.65s/it]
+
+
+  1%|▏         | 10/800 [07:49<6:11:26, 28.21s/it]
+
+
+                                                  
+
+
+{'loss': 0.002, 'grad_norm': 29.42321014404297, 'learning_rate': 5.625e-06, 'num_tokens': 198752.0, 'completions/mean_length': 22.25, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.34375, 'completions/mean_terminated_length': 17.14285659790039, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': -1.015625, 'rewards/format_valid/std': 1.388224482536316, 'rewards/action_legal/mean': -0.65625, 'rewards/action_legal/std': 0.4825586974620819, 'rewards/env_reward/mean': -1.7540408372879028, 'rewards/env_reward/std': 1.7951111793518066, 'rewards/belief_accuracy/mean': -0.07379716634750366, 'rewards/belief_accuracy/std': 0.046562083065509796, 'reward': -2.936046600341797, 'reward_std': 1.997023344039917, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.04934176430106163, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.02}
+
+
+  1%|▏         | 10/800 [07:49<6:11:26, 28.21s/it]
+
+
+  1%|▏         | 11/800 [08:05<5:21:49, 24.47s/it]
+
+
+                                                  
+
+
+{'loss': 0.001, 'grad_norm': 27.308183670043945, 'learning_rate': 6.25e-06, 'num_tokens': 218684.0, 'completions/mean_length': 22.875, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.40625, 'completions/mean_terminated_length': 16.63157844543457, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 28.0, 'rewards/format_valid/mean': -1.203125, 'rewards/format_valid/std': 1.2038873434066772, 'rewards/action_legal/mean': -0.6875, 'rewards/action_legal/std': 0.4709290862083435, 'rewards/env_reward/mean': -2.0874874591827393, 'rewards/env_reward/std': 1.402821660041809, 'rewards/belief_accuracy/mean': -0.09906250238418579, 'rewards/belief_accuracy/std': 0.00530330091714859, 'reward': -3.5229499340057373, 'reward_std': 2.225555419921875, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.02474685851484537, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.02}
+
+
+  1%|▏         | 11/800 [08:05<5:21:49, 24.47s/it]
+
+
+  2%|▏         | 12/800 [08:20<4:47:14, 21.87s/it]
+
+
+                                                  
+
+
+{'loss': 0.0014, 'grad_norm': 83.80093383789062, 'learning_rate': 6.875000000000001e-06, 'num_tokens': 238616.0, 'completions/mean_length': 22.875, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.40625, 'completions/mean_terminated_length': 16.63157844543457, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': -1.453125, 'rewards/format_valid/std': 1.0500335693359375, 'rewards/action_legal/mean': -0.78125, 'rewards/action_legal/std': 0.420013427734375, 'rewards/env_reward/mean': -2.313671827316284, 'rewards/env_reward/std': 1.3194012641906738, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': -3.8822264671325684, 'reward_std': 2.067770004272461, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.03539846441708505, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.02}
+
+
+  2%|▏         | 12/800 [08:20<4:47:14, 21.87s/it]
+
+
+  2%|▏         | 13/800 [08:36<4:23:12, 20.07s/it]
+
+
+                                                  
+
+
+{'loss': 0.0023, 'grad_norm': 46.24530029296875, 'learning_rate': 7.5e-06, 'num_tokens': 258624.0, 'completions/mean_length': 25.25, 'completions/min_length': 9.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.46875, 'completions/mean_terminated_length': 19.294116973876953, 'completions/min_terminated_length': 9.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': -1.671875, 'rewards/format_valid/std': 0.8854502439498901, 'rewards/action_legal/mean': -0.875, 'rewards/action_legal/std': 0.33601075410842896, 'rewards/env_reward/mean': -2.6815249919891357, 'rewards/env_reward/std': 0.8765301704406738, 'rewards/belief_accuracy/mean': -0.10208334028720856, 'rewards/belief_accuracy/std': 0.01178511418402195, 'reward': -4.455881118774414, 'reward_std': 1.1100339889526367, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 0.05627911351621151, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.03}
+
+
+  2%|▏         | 13/800 [08:36<4:23:12, 20.07s/it]
+
+{'loss': 0.0009, 'grad_norm': 79.64603424072266, 'learning_rate': 8.125000000000001e-06, 'num_tokens': 278650.0, 'completions/mean_length': 25.8125, 'completions/min_length': 9.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.5, 'completions/mean_terminated_length': 19.625, 'completions/min_terminated_length': 9.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': -1.4375, 'rewards/format_valid/std': 1.083229899406433, 'rewards/action_legal/mean': -0.78125, 'rewards/action_legal/std': 0.420013427734375, 'rewards/env_reward/mean': -2.406597137451172, 'rewards/env_reward/std': 1.1487802267074585, 'rewards/belief_accuracy/mean': -0.09493793547153473, 'rewards/belief_accuracy/std': 0.02863534912467003, 'reward': -4.005646705627441, 'reward_std': 1.5581319332122803, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 0.02285011764615774, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.03}
+
+
+  2%|▏         | 14/800 [08:52<4:06:25, 18.81s/it]
+
+
+                                                  
+
+
+
+  2%|▏         | 14/800 [08:52<4:06:25, 18.81s/it]
+
+
+  2%|▏         | 15/800 [09:08<3:54:36, 17.93s/it]
+
+
+                                                  
+
+
+{'loss': 0.0019, 'grad_norm': 71.75599670410156, 'learning_rate': 8.75e-06, 'num_tokens': 298566.0, 'completions/mean_length': 22.375, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.40625, 'completions/mean_terminated_length': 15.789473533630371, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 27.0, 'rewards/format_valid/mean': -1.21875, 'rewards/format_valid/std': 1.1773226261138916, 'rewards/action_legal/mean': -0.6875, 'rewards/action_legal/std': 0.4709290862083435, 'rewards/env_reward/mean': -1.9214515686035156, 'rewards/env_reward/std': 1.6459771394729614, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': -3.27748966217041, 'reward_std': 1.699130654335022, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.04659580532461405, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.03}
+
+
+  2%|▏         | 15/800 [09:08<3:54:36, 17.93s/it]
+
+{'loss': 0.0009, 'grad_norm': 37.69881057739258, 'learning_rate': 9.375000000000001e-06, 'num_tokens': 318643.0, 'completions/mean_length': 27.40625, 'completions/min_length': 12.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.5625, 'completions/mean_terminated_length': 21.500001907348633, 'completions/min_terminated_length': 12.0, 'completions/max_terminated_length': 27.0, 'rewards/format_valid/mean': -1.53125, 'rewards/format_valid/std': 0.9913944005966187, 'rewards/action_legal/mean': -0.8125, 'rewards/action_legal/std': 0.3965577781200409, 'rewards/env_reward/mean': -2.4214062690734863, 'rewards/env_reward/std': 1.2254589796066284, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': -4.049296855926514, 'reward_std': 1.9683555364608765, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.02133406139910221, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.03}
+
+
+  2%|▏         | 16/800 [09:24<3:46:23, 17.33s/it]
+
+
+                                                  
+
+
+
+  2%|▏         | 16/800 [09:24<3:46:23, 17.33s/it]
+
+
+  2%|▏         | 17/800 [09:40<3:40:32, 16.90s/it]
+
+
+                                                  
+
+
+{'loss': 0.0014, 'grad_norm': 23.779935836791992, 'learning_rate': 1e-05, 'num_tokens': 338375.0, 'completions/mean_length': 16.625, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.1875, 'completions/mean_terminated_length': 13.076923370361328, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 29.0, 'rewards/format_valid/mean': -0.1875, 'rewards/format_valid/std': 1.2556324005126953, 'rewards/action_legal/mean': -0.3125, 'rewards/action_legal/std': 0.4709290862083435, 'rewards/env_reward/mean': -0.7476803064346313, 'rewards/env_reward/std': 1.5998764038085938, 'rewards/belief_accuracy/mean': -0.1235877200961113, 'rewards/belief_accuracy/std': 0.0713590607047081, 'reward': -1.5172836780548096, 'reward_std': 2.1369173526763916, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.03479788172990084, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.03}
+
+
+  2%|▏         | 17/800 [09:40<3:40:32, 16.90s/it]
+
+
+  2%|▏         | 18/800 [09:56<3:36:22, 16.60s/it]
+
+
+                                                  
+
+
+{'loss': 0.0004, 'grad_norm': 11.554186820983887, 'learning_rate': 1.0625e-05, 'num_tokens': 358277.0, 'completions/mean_length': 21.9375, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.46875, 'completions/mean_terminated_length': 13.058823585510254, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': -1.265625, 'rewards/format_valid/std': 1.2950549125671387, 'rewards/action_legal/mean': -0.75, 'rewards/action_legal/std': 0.4399413466453552, 'rewards/env_reward/mean': -2.0944368839263916, 'rewards/env_reward/std': 1.5971406698226929, 'rewards/belief_accuracy/mean': -0.10863511264324188, 'rewards/belief_accuracy/std': 0.04976717755198479, 'reward': -3.5683417320251465, 'reward_std': 0.9316250085830688, 'frac_reward_zero_std': 0.5, 'completion_length': 32.0, 'kl': 0.010762361343950033, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.04}
+
+
+  2%|▏         | 18/800 [09:56<3:36:22, 16.60s/it]
+
+
+  2%|▏         | 19/800 [10:14<3:42:38, 17.10s/it]
+
+
+                                                  
+
+
+{'loss': 0.0016, 'grad_norm': 21.08062744140625, 'learning_rate': 1.125e-05, 'num_tokens': 377655.0, 'completions/mean_length': 19.5625, 'completions/min_length': 9.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.21875, 'completions/mean_terminated_length': 16.079999923706055, 'completions/min_terminated_length': 9.0, 'completions/max_terminated_length': 27.0, 'rewards/format_valid/mean': -0.40625, 'rewards/format_valid/std': 1.521022081375122, 'rewards/action_legal/mean': -0.46875, 'rewards/action_legal/std': 0.507007360458374, 'rewards/env_reward/mean': -1.0251884460449219, 'rewards/env_reward/std': 1.9070637226104736, 'rewards/belief_accuracy/mean': -0.07682816684246063, 'rewards/belief_accuracy/std': 0.05832352861762047, 'reward': -1.8120169639587402, 'reward_std': 0.658214271068573, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 0.03955374239012599, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.04}
+
+
+  2%|▏         | 19/800 [10:14<3:42:38, 17.10s/it]
+
+{'loss': 0.0019, 'grad_norm': 70.68927764892578, 'learning_rate': 1.1875e-05, 'num_tokens': 397672.0, 'completions/mean_length': 25.53125, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.53125, 'completions/mean_terminated_length': 18.200000762939453, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': -1.359375, 'rewards/format_valid/std': 1.1304750442504883, 'rewards/action_legal/mean': -0.75, 'rewards/action_legal/std': 0.4399413466453552, 'rewards/env_reward/mean': -2.1659765243530273, 'rewards/env_reward/std': 1.4810696840286255, 'rewards/belief_accuracy/mean': -0.10092592239379883, 'rewards/belief_accuracy/std': 0.005237827077507973, 'reward': -3.6572113037109375, 'reward_std': 0.9038525819778442, 'frac_reward_zero_std': 0.5, 'completion_length': 32.0, 'kl': 0.04687770176678896, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.04}
+
+
+  2%|▎         | 20/800 [10:30<3:37:43, 16.75s/it]
+
+
+                                                  
+
+
+
+  2%|▎         | 20/800 [10:30<3:37:43, 16.75s/it]
+
+{'loss': 0.0009, 'grad_norm': 52.50609588623047, 'learning_rate': 1.25e-05, 'num_tokens': 417603.0, 'completions/mean_length': 22.84375, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.40625, 'completions/mean_terminated_length': 16.578947067260742, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': -1.125, 'rewards/format_valid/std': 1.2313119173049927, 'rewards/action_legal/mean': -0.65625, 'rewards/action_legal/std': 0.4825586974620819, 'rewards/env_reward/mean': -1.8219103813171387, 'rewards/env_reward/std': 1.6799882650375366, 'rewards/belief_accuracy/mean': -0.10521601140499115, 'rewards/belief_accuracy/std': 0.029506206512451172, 'reward': -3.137575626373291, 'reward_std': 1.4506244659423828, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 0.023099651327356696, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.04}
+
+
+  3%|▎         | 21/800 [10:46<3:34:10, 16.50s/it]
+
+
+                                                  
+
+
+
+  3%|▎         | 21/800 [10:46<3:34:10, 16.50s/it]
+
+{'loss': 0.0019, 'grad_norm': 15.835291862487793, 'learning_rate': 1.3125e-05, 'num_tokens': 437574.0, 'completions/mean_length': 24.09375, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.375, 'completions/mean_terminated_length': 19.350000381469727, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': -1.59375, 'rewards/format_valid/std': 0.9625025987625122, 'rewards/action_legal/mean': -0.84375, 'rewards/action_legal/std': 0.3689020276069641, 'rewards/env_reward/mean': -2.503159523010254, 'rewards/env_reward/std': 1.1811230182647705, 'rewards/belief_accuracy/mean': -0.10104167461395264, 'rewards/belief_accuracy/std': 0.005892557557672262, 'reward': -4.179738998413086, 'reward_std': 1.5668916702270508, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 0.04641182720661163, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.04}
+
+
+  3%|▎         | 22/800 [11:02<3:31:37, 16.32s/it]
+
+
+                                                  
+
+
+
+  3%|▎         | 22/800 [11:02<3:31:37, 16.32s/it]
+
+{'loss': 0.0034, 'grad_norm': 19.302196502685547, 'learning_rate': 1.3750000000000002e-05, 'num_tokens': 457517.0, 'completions/mean_length': 23.21875, 'completions/min_length': 9.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.46875, 'completions/mean_terminated_length': 15.470588684082031, 'completions/min_terminated_length': 9.0, 'completions/max_terminated_length': 29.0, 'rewards/format_valid/mean': -0.953125, 'rewards/format_valid/std': 1.3816736936569214, 'rewards/action_legal/mean': -0.625, 'rewards/action_legal/std': 0.49186936020851135, 'rewards/env_reward/mean': -1.7773826122283936, 'rewards/env_reward/std': 1.6126797199249268, 'rewards/belief_accuracy/mean': -0.08421479165554047, 'rewards/belief_accuracy/std': 0.04115736857056618, 'reward': -2.997624397277832, 'reward_std': 1.9654653072357178, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.08415555115789175, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.05}
+
+
+  3%|▎         | 23/800 [11:18<3:29:43, 16.20s/it]
+
+
+                                                  
+
+
+
+  3%|▎         | 23/800 [11:18<3:29:43, 16.20s/it]
+
+
+  3%|▎         | 24/800 [11:34<3:28:19, 16.11s/it]
+
+
+                                                  
+
+
+{'loss': 0.0017, 'grad_norm': 85.42478942871094, 'learning_rate': 1.4374999999999999e-05, 'num_tokens': 477531.0, 'completions/mean_length': 25.4375, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.53125, 'completions/mean_terminated_length': 18.0, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': -1.4375, 'rewards/format_valid/std': 1.083229899406433, 'rewards/action_legal/mean': -0.78125, 'rewards/action_legal/std': 0.420013427734375, 'rewards/env_reward/mean': -2.2515881061553955, 'rewards/env_reward/std': 1.449811339378357, 'rewards/belief_accuracy/mean': -0.0978076308965683, 'rewards/belief_accuracy/std': 0.012401958927512169, 'reward': -3.781742572784424, 'reward_std': 2.3154501914978027, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.041403453797101974, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.05}
+
+
+  3%|▎         | 24/800 [11:34<3:28:19, 16.11s/it]
+
+{'loss': 0.0022, 'grad_norm': 31.280765533447266, 'learning_rate': 1.5e-05, 'num_tokens': 497602.0, 'completions/mean_length': 27.21875, 'completions/min_length': 9.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.65625, 'completions/mean_terminated_length': 18.090909957885742, 'completions/min_terminated_length': 9.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': -1.765625, 'rewards/format_valid/std': 0.7403614521026611, 'rewards/action_legal/mean': -0.90625, 'rewards/action_legal/std': 0.2961445748806, 'rewards/env_reward/mean': -2.6920697689056396, 'rewards/env_reward/std': 0.9738791584968567, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': -4.47169828414917, 'reward_std': 1.3528409004211426, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 0.05614800192415714, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.05}
+
+
+  3%|▎         | 25/800 [11:50<3:27:16, 16.05s/it]
+
+
+                                                  
+
+
+
+  3%|▎         | 25/800 [11:50<3:27:16, 16.05s/it]
+
+
+  3%|▎         | 26/800 [12:06<3:26:35, 16.01s/it]
+
+
+                                                  
+
+
+{'loss': 0.0014, 'grad_norm': 46.4079475402832, 'learning_rate': 1.5625e-05, 'num_tokens': 517727.0, 'completions/mean_length': 28.90625, 'completions/min_length': 12.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.71875, 'completions/mean_terminated_length': 21.0, 'completions/min_terminated_length': 12.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': -1.53125, 'rewards/format_valid/std': 0.9913944005966187, 'rewards/action_legal/mean': -0.8125, 'rewards/action_legal/std': 0.3965577781200409, 'rewards/env_reward/mean': -2.4508020877838135, 'rewards/env_reward/std': 1.174810767173767, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': -4.093390464782715, 'reward_std': 1.8875445127487183, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.034114168491214514, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.05}
+
+
+  3%|▎         | 26/800 [12:06<3:26:35, 16.01s/it]
+
+
+  3%|▎         | 27/800 [12:21<3:25:56, 15.98s/it]
+
+
+                                                  
+
+
+{'loss': 0.003, 'grad_norm': 84.28619384765625, 'learning_rate': 1.6250000000000002e-05, 'num_tokens': 537787.0, 'completions/mean_length': 26.875, 'completions/min_length': 9.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.625, 'completions/mean_terminated_length': 18.33333396911621, 'completions/min_terminated_length': 9.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': -1.765625, 'rewards/format_valid/std': 0.7403614521026611, 'rewards/action_legal/mean': -0.90625, 'rewards/action_legal/std': 0.2961445748806, 'rewards/env_reward/mean': -2.7157158851623535, 'rewards/env_reward/std': 0.9025736451148987, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': -4.507167816162109, 'reward_std': 1.2525182962417603, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 0.07624564506113529, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.05}
+
+
+  3%|▎         | 27/800 [12:21<3:25:56, 15.98s/it]
+
+{'loss': 0.0062, 'grad_norm': 12.795799255371094, 'learning_rate': 1.6875000000000004e-05, 'num_tokens': 557703.0, 'completions/mean_length': 22.375, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.34375, 'completions/mean_terminated_length': 17.33333396911621, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': -0.890625, 'rewards/format_valid/std': 1.3721561431884766, 'rewards/action_legal/mean': -0.59375, 'rewards/action_legal/std': 0.49899089336395264, 'rewards/env_reward/mean': -1.6386311054229736, 'rewards/env_reward/std': 1.690489649772644, 'rewards/belief_accuracy/mean': -0.12766025960445404, 'rewards/belief_accuracy/std': 0.06216808781027794, 'reward': -2.9151458740234375, 'reward_std': 1.7658030986785889, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 0.15617628395557404, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.06}
+
+
+  4%|▎         | 28/800 [12:37<3:25:27, 15.97s/it]
+
+
+                                                  
+
+
+
+  4%|▎         | 28/800 [12:37<3:25:27, 15.97s/it]
+
+
+  4%|▎         | 29/800 [12:53<3:24:55, 15.95s/it]
+
+
+                                                  
+
+
+{'loss': 0.009, 'grad_norm': 31.969526290893555, 'learning_rate': 1.75e-05, 'num_tokens': 577461.0, 'completions/mean_length': 17.4375, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.25, 'completions/mean_terminated_length': 12.583333969116211, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 28.0, 'rewards/format_valid/mean': -0.375, 'rewards/format_valid/std': 1.3795745372772217, 'rewards/action_legal/mean': -0.40625, 'rewards/action_legal/std': 0.49899089336395264, 'rewards/env_reward/mean': -1.2202446460723877, 'rewards/env_reward/std': 1.5190094709396362, 'rewards/belief_accuracy/mean': -0.11423526704311371, 'rewards/belief_accuracy/std': 0.0589185394346714, 'reward': -2.212135076522827, 'reward_std': 1.2390978336334229, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.22568923607468605, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.06}
+
+
+  4%|▎         | 29/800 [12:53<3:24:55, 15.95s/it]
+
+
+  4%|▍         | 30/800 [13:09<3:24:29, 15.93s/it]
+
+
+                                                  
+
+
+{'loss': 0.0052, 'grad_norm': 5.53366231918335, 'learning_rate': 1.8125e-05, 'num_tokens': 597482.0, 'completions/mean_length': 25.65625, 'completions/min_length': 9.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.46875, 'completions/mean_terminated_length': 20.05882453918457, 'completions/min_terminated_length': 9.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': -1.53125, 'rewards/format_valid/std': 0.9913944005966187, 'rewards/action_legal/mean': -0.8125, 'rewards/action_legal/std': 0.3965577781200409, 'rewards/env_reward/mean': -2.4067306518554688, 'rewards/env_reward/std': 1.2677935361862183, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': -4.027283191680908, 'reward_std': 1.6862303018569946, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 0.12904555909335613, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.06}
+
+
+  4%|▍         | 30/800 [13:09<3:24:29, 15.93s/it]
+
+
+  4%|▍         | 31/800 [13:25<3:24:09, 15.93s/it]
+
+
+                                                  
+
+
+{'loss': 0.0068, 'grad_norm': 59.43049240112305, 'learning_rate': 1.8750000000000002e-05, 'num_tokens': 617426.0, 'completions/mean_length': 23.25, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.375, 'completions/mean_terminated_length': 18.0, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': -1.375, 'rewards/format_valid/std': 1.099853277206421, 'rewards/action_legal/mean': -0.75, 'rewards/action_legal/std': 0.4399413466453552, 'rewards/env_reward/mean': -2.212999105453491, 'rewards/env_reward/std': 1.3898619413375854, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': -3.7257485389709473, 'reward_std': 2.2722411155700684, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.17105461843311787, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.06}
+
+
+  4%|▍         | 31/800 [13:25<3:24:09, 15.93s/it]
+
+
+  4%|▍         | 32/800 [13:41<3:23:51, 15.93s/it]
+
+
+                                                  
+
+
+{'loss': 0.0103, 'grad_norm': 48.81121063232422, 'learning_rate': 1.9375e-05, 'num_tokens': 637261.0, 'completions/mean_length': 19.84375, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.21875, 'completions/mean_terminated_length': 16.440000534057617, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': -0.9375, 'rewards/format_valid/std': 1.4013242721557617, 'rewards/action_legal/mean': -0.625, 'rewards/action_legal/std': 0.49186936020851135, 'rewards/env_reward/mean': -1.6380008459091187, 'rewards/env_reward/std': 1.8035916090011597, 'rewards/belief_accuracy/mean': -0.0928797498345375, 'rewards/belief_accuracy/std': 0.046385712921619415, 'reward': -2.813765525817871, 'reward_std': 1.6431068181991577, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 0.25662683695554733, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.06}
+
+
+  4%|▍         | 32/800 [13:41<3:23:51, 15.93s/it]
+
+
+  4%|▍         | 33/800 [13:57<3:23:33, 15.92s/it]
+
+
+                                                  
+
+
+{'loss': 0.0061, 'grad_norm': 24.548816680908203, 'learning_rate': 2e-05, 'num_tokens': 657292.0, 'completions/mean_length': 25.96875, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.625, 'completions/mean_terminated_length': 15.916666984558105, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': -1.453125, 'rewards/format_valid/std': 1.0500335693359375, 'rewards/action_legal/mean': -0.78125, 'rewards/action_legal/std': 0.420013427734375, 'rewards/env_reward/mean': -2.278764247894287, 'rewards/env_reward/std': 1.3933061361312866, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': -3.829864978790283, 'reward_std': 2.1860780715942383, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.1519254334270954, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.07}
+
+
+  4%|▍         | 33/800 [13:57<3:23:33, 15.92s/it]
+
+
+  4%|▍         | 34/800 [14:13<3:23:13, 15.92s/it]
+
+
+                                                  
+
+
+{'loss': 0.0086, 'grad_norm': 62.92240905761719, 'learning_rate': 2.0625e-05, 'num_tokens': 677243.0, 'completions/mean_length': 23.46875, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.5, 'completions/mean_terminated_length': 14.9375, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 29.0, 'rewards/format_valid/mean': -1.4375, 'rewards/format_valid/std': 1.083229899406433, 'rewards/action_legal/mean': -0.78125, 'rewards/action_legal/std': 0.420013427734375, 'rewards/env_reward/mean': -2.3169703483581543, 'rewards/env_reward/std': 1.3146628141403198, 'rewards/belief_accuracy/mean': -0.10208334028720856, 'rewards/belief_accuracy/std': 0.01178511418402195, 'reward': -3.8926427364349365, 'reward_std': 2.0074849128723145, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.21463509276509285, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.07}
+
+
+  4%|▍         | 34/800 [14:13<3:23:13, 15.92s/it]
+
+{'loss': 0.0105, 'grad_norm': 111.84656524658203, 'learning_rate': 2.125e-05, 'num_tokens': 697226.0, 'completions/mean_length': 24.46875, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.46875, 'completions/mean_terminated_length': 17.823530197143555, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 28.0, 'rewards/format_valid/mean': -1.125, 'rewards/format_valid/std': 1.2313119173049927, 'rewards/action_legal/mean': -0.65625, 'rewards/action_legal/std': 0.4825586974620819, 'rewards/env_reward/mean': -1.8845045566558838, 'rewards/env_reward/std': 1.5734351873397827, 'rewards/belief_accuracy/mean': -0.09748905897140503, 'rewards/belief_accuracy/std': 0.014204003848135471, 'reward': -3.2082862854003906, 'reward_std': 2.4869072437286377, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.26274506747722626, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.07}
+
+
+  4%|▍         | 35/800 [14:29<3:23:04, 15.93s/it]
+
+
+                                                  
+
+
+
+  4%|▍         | 35/800 [14:29<3:23:04, 15.93s/it]
+
+
+  4%|▍         | 36/800 [14:45<3:22:47, 15.93s/it]
+
+
+                                                  
+
+
+{'loss': 0.0085, 'grad_norm': 10.248196601867676, 'learning_rate': 2.1875e-05, 'num_tokens': 717318.0, 'completions/mean_length': 27.875, 'completions/min_length': 11.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.65625, 'completions/mean_terminated_length': 20.0, 'completions/min_terminated_length': 11.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': -1.84375, 'rewards/format_valid/std': 0.6148366928100586, 'rewards/action_legal/mean': -0.9375, 'rewards/action_legal/std': 0.24593468010425568, 'rewards/env_reward/mean': -2.753528118133545, 'rewards/env_reward/std': 0.9749595522880554, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': -4.569355010986328, 'reward_std': 1.076627254486084, 'frac_reward_zero_std': 0.5, 'completion_length': 32.0, 'kl': 0.21304048970341682, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.07}
+
+
+  4%|▍         | 36/800 [14:45<3:22:47, 15.93s/it]
+
+
+  5%|▍         | 37/800 [15:01<3:22:36, 15.93s/it]
+
+
+                                                  
+
+
+{'loss': 0.0142, 'grad_norm': 31.553918838500977, 'learning_rate': 2.25e-05, 'num_tokens': 737293.0, 'completions/mean_length': 24.21875, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.4375, 'completions/mean_terminated_length': 18.16666603088379, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 29.0, 'rewards/format_valid/mean': -0.78125, 'rewards/format_valid/std': 1.3255400657653809, 'rewards/action_legal/mean': -0.53125, 'rewards/action_legal/std': 0.507007360458374, 'rewards/env_reward/mean': -1.485032320022583, 'rewards/env_reward/std': 1.6505156755447388, 'rewards/belief_accuracy/mean': -0.09892420470714569, 'rewards/belief_accuracy/std': 0.02798508107662201, 'reward': -2.5899457931518555, 'reward_std': 2.5904335975646973, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.354715496301651, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.07}
+
+
+  5%|▍         | 37/800 [15:01<3:22:36, 15.93s/it]
+
+
+  5%|▍         | 38/800 [15:17<3:22:16, 15.93s/it]
+
+
+                                                  
+
+
+{'loss': 0.0227, 'grad_norm': 8.331291198730469, 'learning_rate': 2.3125000000000003e-05, 'num_tokens': 757111.0, 'completions/mean_length': 19.3125, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.25, 'completions/mean_terminated_length': 15.083333969116211, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 27.0, 'rewards/format_valid/mean': -0.90625, 'rewards/format_valid/std': 1.260040283203125, 'rewards/action_legal/mean': -0.5625, 'rewards/action_legal/std': 0.504016101360321, 'rewards/env_reward/mean': -1.6277079582214355, 'rewards/env_reward/std': 1.5928757190704346, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': -2.8149993419647217, 'reward_std': 2.3546388149261475, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.5667486786842346, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.08}
+
+
+  5%|▍         | 38/800 [15:17<3:22:16, 15.93s/it]
+
+
+  5%|▍         | 39/800 [15:33<3:22:04, 15.93s/it]
+
+
+                                                  
+
+
+{'loss': 0.017, 'grad_norm': 10.413817405700684, 'learning_rate': 2.375e-05, 'num_tokens': 777078.0, 'completions/mean_length': 23.96875, 'completions/min_length': 9.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.46875, 'completions/mean_terminated_length': 16.882352828979492, 'completions/min_terminated_length': 9.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': -0.671875, 'rewards/format_valid/std': 1.359609603881836, 'rewards/action_legal/mean': -0.5, 'rewards/action_legal/std': 0.5080004930496216, 'rewards/env_reward/mean': -1.4943370819091797, 'rewards/env_reward/std': 1.5417914390563965, 'rewards/belief_accuracy/mean': -0.0980014055967331, 'rewards/belief_accuracy/std': 0.03148852661252022, 'reward': -2.5941033363342285, 'reward_std': 2.3718578815460205, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.42607562988996506, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.08}
+
+
+  5%|▍         | 39/800 [15:33<3:22:04, 15.93s/it]
+
+{'loss': 0.0323, 'grad_norm': 49.17258071899414, 'learning_rate': 2.4375e-05, 'num_tokens': 796398.0, 'completions/mean_length': 17.75, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.125, 'completions/mean_terminated_length': 15.714286804199219, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': -0.71875, 'rewards/format_valid/std': 1.3966405391693115, 'rewards/action_legal/mean': -0.53125, 'rewards/action_legal/std': 0.507007360458374, 'rewards/env_reward/mean': -1.7342383861541748, 'rewards/env_reward/std': 1.463059663772583, 'rewards/belief_accuracy/mean': -0.078125, 'rewards/belief_accuracy/std': 0.04200134426355362, 'reward': -2.8982324600219727, 'reward_std': 2.035499095916748, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.8074239492416382, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.08}
+
+
+  5%|▌         | 40/800 [15:51<3:30:43, 16.64s/it]
+
+
+                                                  
+
+
+
+  5%|▌         | 40/800 [15:51<3:30:43, 16.64s/it]
+
+{'loss': 0.0403, 'grad_norm': 25.979249954223633, 'learning_rate': 2.5e-05, 'num_tokens': 815789.0, 'completions/mean_length': 19.96875, 'completions/min_length': 9.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.25, 'completions/mean_terminated_length': 15.958333969116211, 'completions/min_terminated_length': 9.0, 'completions/max_terminated_length': 29.0, 'rewards/format_valid/mean': -0.546875, 'rewards/format_valid/std': 1.3990744352340698, 'rewards/action_legal/mean': -0.46875, 'rewards/action_legal/std': 0.507007360458374, 'rewards/env_reward/mean': -1.2628259658813477, 'rewards/env_reward/std': 1.6710240840911865, 'rewards/belief_accuracy/mean': -0.07500000298023224, 'rewards/belief_accuracy/std': 0.04399413242936134, 'reward': -2.17002010345459, 'reward_std': 2.517836093902588, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.0078506469726562, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.08}
+
+
+  5%|▌         | 41/800 [16:09<3:36:46, 17.14s/it]
+
+
+                                                  
+
+
+
+  5%|▌         | 41/800 [16:09<3:36:46, 17.14s/it]
+
+
+  5%|▌         | 42/800 [16:25<3:31:59, 16.78s/it]
+
+
+                                                  
+
+
+{'loss': 0.0306, 'grad_norm': 44.95201110839844, 'learning_rate': 2.5625e-05, 'num_tokens': 835723.0, 'completions/mean_length': 22.9375, 'completions/min_length': 9.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.375, 'completions/mean_terminated_length': 17.5, 'completions/min_terminated_length': 9.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': -0.640625, 'rewards/format_valid/std': 1.3028156757354736, 'rewards/action_legal/mean': -0.46875, 'rewards/action_legal/std': 0.507007360458374, 'rewards/env_reward/mean': -1.3257067203521729, 'rewards/env_reward/std': 1.6000605821609497, 'rewards/belief_accuracy/mean': -0.09982964396476746, 'rewards/belief_accuracy/std': 0.0176384299993515, 'reward': -2.343517780303955, 'reward_std': 2.502542495727539, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.7656729966402054, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.08}
+
+
+  5%|▌         | 42/800 [16:25<3:31:59, 16.78s/it]
+
+
+  5%|▌         | 43/800 [16:42<3:31:40, 16.78s/it]
+
+
+                                                  
+
+
+{'loss': 0.0339, 'grad_norm': 48.966094970703125, 'learning_rate': 2.625e-05, 'num_tokens': 855563.0, 'completions/mean_length': 20.0, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.375, 'completions/mean_terminated_length': 12.800000190734863, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': -0.71875, 'rewards/format_valid/std': 1.30716073513031, 'rewards/action_legal/mean': -0.5, 'rewards/action_legal/std': 0.5080004930496216, 'rewards/env_reward/mean': -1.4830719232559204, 'rewards/env_reward/std': 1.5826469659805298, 'rewards/belief_accuracy/mean': -0.0972222238779068, 'rewards/belief_accuracy/std': 0.030264990404248238, 'reward': -2.577211856842041, 'reward_std': 2.207462787628174, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.8472781032323837, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.09}
+
+
+  5%|▌         | 43/800 [16:42<3:31:40, 16.78s/it]
+
+{'loss': 0.0288, 'grad_norm': 69.83633422851562, 'learning_rate': 2.6875e-05, 'num_tokens': 875392.0, 'completions/mean_length': 19.65625, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.34375, 'completions/mean_terminated_length': 13.190476417541504, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': -0.234375, 'rewards/format_valid/std': 1.3136035203933716, 'rewards/action_legal/mean': -0.34375, 'rewards/action_legal/std': 0.4825586974620819, 'rewards/env_reward/mean': -0.6458324193954468, 'rewards/env_reward/std': 1.7775462865829468, 'rewards/belief_accuracy/mean': -0.03386639803647995, 'rewards/belief_accuracy/std': 0.11961784958839417, 'reward': -1.0992541313171387, 'reward_std': 2.028806447982788, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.7195403873920441, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.09}
+
+
+  6%|▌         | 44/800 [16:58<3:28:05, 16.51s/it]
+
+
+                                                  
+
+
+
+  6%|▌         | 44/800 [16:58<3:28:05, 16.51s/it]
+
+
+  6%|▌         | 45/800 [17:14<3:25:49, 16.36s/it]
+
+
+                                                  
+
+
+{'loss': 0.0436, 'grad_norm': 3.0112521648406982, 'learning_rate': 2.7500000000000004e-05, 'num_tokens': 895225.0, 'completions/mean_length': 19.78125, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.21875, 'completions/mean_terminated_length': 16.35999870300293, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': -0.234375, 'rewards/format_valid/std': 1.2180389165878296, 'rewards/action_legal/mean': -0.3125, 'rewards/action_legal/std': 0.4709290862083435, 'rewards/env_reward/mean': -0.8469383716583252, 'rewards/env_reward/std': 1.4986060857772827, 'rewards/belief_accuracy/mean': -0.09856715798377991, 'rewards/belief_accuracy/std': 0.02865961380302906, 'reward': -1.5934526920318604, 'reward_std': 2.1515111923217773, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.088939145207405, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.09}
+
+
+  6%|▌         | 45/800 [17:14<3:25:49, 16.36s/it]
+
+{'loss': 0.0366, 'grad_norm': 12.296052932739258, 'learning_rate': 2.8125000000000003e-05, 'num_tokens': 915083.0, 'completions/mean_length': 20.5625, 'completions/min_length': 10.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.1875, 'completions/mean_terminated_length': 17.923078536987305, 'completions/min_terminated_length': 10.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': -0.359375, 'rewards/format_valid/std': 1.3028156757354736, 'rewards/action_legal/mean': -0.375, 'rewards/action_legal/std': 0.49186936020851135, 'rewards/env_reward/mean': -0.8200122117996216, 'rewards/env_reward/std': 1.7326771020889282, 'rewards/belief_accuracy/mean': -0.09766387194395065, 'rewards/belief_accuracy/std': 0.04987531527876854, 'reward': -1.5597286224365234, 'reward_std': 2.4947471618652344, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.915252074599266, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.09}
+
+
+  6%|▌         | 46/800 [17:30<3:23:50, 16.22s/it]
+
+
+                                                  
+
+
+
+  6%|▌         | 46/800 [17:30<3:23:50, 16.22s/it]
+
+
+  6%|▌         | 47/800 [17:46<3:22:30, 16.14s/it]
+
+
+                                                  
+
+
+{'loss': 0.0319, 'grad_norm': 19.27388572692871, 'learning_rate': 2.8749999999999997e-05, 'num_tokens': 934960.0, 'completions/mean_length': 21.15625, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.34375, 'completions/mean_terminated_length': 15.476190567016602, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': -0.34375, 'rewards/format_valid/std': 1.221035361289978, 'rewards/action_legal/mean': -0.34375, 'rewards/action_legal/std': 0.4825586974620819, 'rewards/env_reward/mean': -0.9945231080055237, 'rewards/env_reward/std': 1.5367543697357178, 'rewards/belief_accuracy/mean': -0.09874999523162842, 'rewards/belief_accuracy/std': 0.0070710680447518826, 'reward': -1.8224096298217773, 'reward_std': 2.3957011699676514, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.7979511916637421, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.09}
+
+
+  6%|▌         | 47/800 [17:46<3:22:30, 16.14s/it]
+
+{'loss': 0.0342, 'grad_norm': 7.057971000671387, 'learning_rate': 2.9375000000000003e-05, 'num_tokens': 954930.0, 'completions/mean_length': 24.0625, 'completions/min_length': 5.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.40625, 'completions/mean_terminated_length': 18.63157844543457, 'completions/min_terminated_length': 5.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': -0.734375, 'rewards/format_valid/std': 1.2888127565383911, 'rewards/action_legal/mean': -0.5, 'rewards/action_legal/std': 0.5080004930496216, 'rewards/env_reward/mean': -1.4825866222381592, 'rewards/env_reward/std': 1.5664743185043335, 'rewards/belief_accuracy/mean': -0.09534955769777298, 'rewards/belief_accuracy/std': 0.0263068787753582, 'reward': -2.5716474056243896, 'reward_std': 2.4487414360046387, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.8540289849042892, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.1}
+
+
+  6%|▌         | 48/800 [18:02<3:21:37, 16.09s/it]
+
+
+                                                  
+
+
+
+  6%|▌         | 48/800 [18:02<3:21:37, 16.09s/it]
+
+{'loss': 0.0441, 'grad_norm': 34.03550720214844, 'learning_rate': 3e-05, 'num_tokens': 974743.0, 'completions/mean_length': 19.15625, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.3125, 'completions/mean_terminated_length': 13.318181991577148, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 27.0, 'rewards/format_valid/mean': -0.203125, 'rewards/format_valid/std': 1.243430256843567, 'rewards/action_legal/mean': -0.3125, 'rewards/action_legal/std': 0.4709290862083435, 'rewards/env_reward/mean': -0.8020281791687012, 'rewards/env_reward/std': 1.5413240194320679, 'rewards/belief_accuracy/mean': -0.09811699390411377, 'rewards/belief_accuracy/std': 0.025741804391145706, 'reward': -1.5231744050979614, 'reward_std': 2.1568450927734375, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.1035194247961044, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.1}
+
+
+  6%|▌         | 49/800 [18:18<3:21:01, 16.06s/it]
+
+
+                                                  
+
+
+
+  6%|▌         | 49/800 [18:18<3:21:01, 16.06s/it]
+
+
+  6%|▋         | 50/800 [18:33<3:20:21, 16.03s/it]
+
+
+                                                  
+
+
+{'loss': 0.0241, 'grad_norm': 4.326381683349609, 'learning_rate': 3.0625000000000006e-05, 'num_tokens': 994687.0, 'completions/mean_length': 23.25, 'completions/min_length': 9.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.46875, 'completions/mean_terminated_length': 15.529412269592285, 'completions/min_terminated_length': 9.0, 'completions/max_terminated_length': 27.0, 'rewards/format_valid/mean': -0.59375, 'rewards/format_valid/std': 1.260040283203125, 'rewards/action_legal/mean': -0.4375, 'rewards/action_legal/std': 0.504016101360321, 'rewards/env_reward/mean': -1.3798023462295532, 'rewards/env_reward/std': 1.4862309694290161, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': -2.4212660789489746, 'reward_std': 2.3536713123321533, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.6022636666893959, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.1}
+
+
+  6%|▋         | 50/800 [18:33<3:20:21, 16.03s/it]
+
+{'loss': 0.0341, 'grad_norm': 7.270224094390869, 'learning_rate': 3.125e-05, 'num_tokens': 1014579.0, 'completions/mean_length': 21.625, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.40625, 'completions/mean_terminated_length': 14.526315689086914, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 27.0, 'rewards/format_valid/mean': -0.328125, 'rewards/format_valid/std': 1.235296607017517, 'rewards/action_legal/mean': -0.34375, 'rewards/action_legal/std': 0.4825586974620819, 'rewards/env_reward/mean': -0.8408621549606323, 'rewards/env_reward/std': 1.6079081296920776, 'rewards/belief_accuracy/mean': -0.10884296894073486, 'rewards/belief_accuracy/std': 0.047710828483104706, 'reward': -1.6214157342910767, 'reward_std': 1.9243780374526978, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.8536929339170456, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.1}
+
+
+  6%|▋         | 51/800 [18:49<3:19:51, 16.01s/it]
+
+
+                                                  
+
+
+
+  6%|▋         | 51/800 [18:49<3:19:51, 16.01s/it]
+
+
+  6%|▋         | 52/800 [19:05<3:19:23, 15.99s/it]
+
+
+                                                  
+
+
+{'loss': 0.025, 'grad_norm': 2.841190814971924, 'learning_rate': 3.1875e-05, 'num_tokens': 1034616.0, 'completions/mean_length': 26.15625, 'completions/min_length': 9.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.625, 'completions/mean_terminated_length': 16.416667938232422, 'completions/min_terminated_length': 9.0, 'completions/max_terminated_length': 27.0, 'rewards/format_valid/mean': -0.65625, 'rewards/format_valid/std': 1.2853862047195435, 'rewards/action_legal/mean': -0.46875, 'rewards/action_legal/std': 0.507007360458374, 'rewards/env_reward/mean': -1.391495704650879, 'rewards/env_reward/std': 1.5563961267471313, 'rewards/belief_accuracy/mean': -0.09626838564872742, 'rewards/belief_accuracy/std': 0.021109245717525482, 'reward': -2.4322986602783203, 'reward_std': 2.3410863876342773, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.625769779086113, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.1}
+
+
+  6%|▋         | 52/800 [19:05<3:19:23, 15.99s/it]
+
+{'loss': 0.0363, 'grad_norm': 4.692880630493164, 'learning_rate': 3.2500000000000004e-05, 'num_tokens': 1054457.0, 'completions/mean_length': 20.03125, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.21875, 'completions/mean_terminated_length': 16.68000030517578, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.234375, 'rewards/format_valid/std': 0.9998739361763, 'rewards/action_legal/mean': -0.15625, 'rewards/action_legal/std': 0.3689020276069641, 'rewards/env_reward/mean': -0.5063405632972717, 'rewards/env_reward/std': 1.155798077583313, 'rewards/belief_accuracy/mean': -0.09147512167692184, 'rewards/belief_accuracy/std': 0.0467376746237278, 'reward': -1.0300300121307373, 'reward_std': 1.5400068759918213, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.9076180011034012, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.11}
+
+
+  7%|▋         | 53/800 [19:21<3:18:53, 15.98s/it]
+
+
+                                                  
+
+
+
+  7%|▋         | 53/800 [19:21<3:18:53, 15.98s/it]
+
+{'loss': 0.0335, 'grad_norm': 5.916451454162598, 'learning_rate': 3.3125e-05, 'num_tokens': 1073851.0, 'completions/mean_length': 20.0625, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.3125, 'completions/mean_terminated_length': 14.636363983154297, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': -0.015625, 'rewards/format_valid/std': 1.2793923616409302, 'rewards/action_legal/mean': -0.28125, 'rewards/action_legal/std': 0.45680341124534607, 'rewards/env_reward/mean': -0.4909473955631256, 'rewards/env_reward/std': 1.6443326473236084, 'rewards/belief_accuracy/mean': -0.05921213701367378, 'rewards/belief_accuracy/std': 0.06019812077283859, 'reward': -0.9289012551307678, 'reward_std': 1.8463006019592285, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.8374583423137665, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.11}
+
+
+  7%|▋         | 54/800 [19:40<3:27:17, 16.67s/it]
+
+
+                                                  
+
+
+
+  7%|▋         | 54/800 [19:40<3:27:17, 16.67s/it]
+
+
+  7%|▋         | 55/800 [19:56<3:24:22, 16.46s/it]
+
+
+                                                  
+
+
+{'loss': 0.0399, 'grad_norm': 5.239197731018066, 'learning_rate': 3.375000000000001e-05, 'num_tokens': 1093762.0, 'completions/mean_length': 22.21875, 'completions/min_length': 9.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.3125, 'completions/mean_terminated_length': 17.772727966308594, 'completions/min_terminated_length': 9.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.28125, 'rewards/format_valid/std': 0.8974650502204895, 'rewards/action_legal/mean': -0.125, 'rewards/action_legal/std': 0.33601075410842896, 'rewards/env_reward/mean': -0.23652157187461853, 'rewards/env_reward/std': 1.0832096338272095, 'rewards/belief_accuracy/mean': -0.1000327467918396, 'rewards/belief_accuracy/std': 0.030893482267856598, 'reward': -0.6470681428909302, 'reward_std': 1.5397109985351562, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.997236430644989, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.11}
+
+
+  7%|▋         | 55/800 [19:56<3:24:22, 16.46s/it]
+
+{'loss': 0.0449, 'grad_norm': 12.118021965026855, 'learning_rate': 3.4375e-05, 'num_tokens': 1113555.0, 'completions/mean_length': 18.53125, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.1875, 'completions/mean_terminated_length': 15.423077583312988, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': -0.140625, 'rewards/format_valid/std': 1.192952275276184, 'rewards/action_legal/mean': -0.28125, 'rewards/action_legal/std': 0.45680341124534607, 'rewards/env_reward/mean': -1.0201691389083862, 'rewards/env_reward/std': 1.2754989862442017, 'rewards/belief_accuracy/mean': -0.10769675672054291, 'rewards/belief_accuracy/std': 0.020930759608745575, 'reward': -1.8744375705718994, 'reward_std': 1.8477702140808105, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.1232753843069077, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.11}
+
+
+  7%|▋         | 56/800 [20:12<3:22:17, 16.31s/it]
+
+
+                                                  
+
+
+
+  7%|▋         | 56/800 [20:12<3:22:17, 16.31s/it]
+
+
+  7%|▋         | 57/800 [20:28<3:20:41, 16.21s/it]
+
+
+                                                  
+
+
+{'loss': 0.0705, 'grad_norm': 24.4376220703125, 'learning_rate': 3.5e-05, 'num_tokens': 1133362.0, 'completions/mean_length': 18.96875, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.28125, 'completions/mean_terminated_length': 13.8695650100708, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 28.0, 'rewards/format_valid/mean': 0.21875, 'rewards/format_valid/std': 0.8607714176177979, 'rewards/action_legal/mean': -0.125, 'rewards/action_legal/std': 0.33601075410842896, 'rewards/env_reward/mean': -0.22579313814640045, 'rewards/env_reward/std': 1.0856302976608276, 'rewards/belief_accuracy/mean': -0.09614583849906921, 'rewards/belief_accuracy/std': 0.01757950708270073, 'reward': -0.6224396824836731, 'reward_std': 1.0715909004211426, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.7623091340065002, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.11}
+
+
+  7%|▋         | 57/800 [20:28<3:20:41, 16.21s/it]
+
+{'loss': 0.0379, 'grad_norm': 4.424113750457764, 'learning_rate': 3.5625000000000005e-05, 'num_tokens': 1153227.0, 'completions/mean_length': 20.78125, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.3125, 'completions/mean_terminated_length': 15.681818962097168, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.046875, 'rewards/format_valid/std': 1.0028939247131348, 'rewards/action_legal/mean': -0.1875, 'rewards/action_legal/std': 0.3965577781200409, 'rewards/env_reward/mean': -0.3962726593017578, 'rewards/env_reward/std': 1.3369419574737549, 'rewards/belief_accuracy/mean': -0.1003125011920929, 'rewards/belief_accuracy/std': 0.0017677665455266833, 'reward': -0.9023776650428772, 'reward_std': 1.9154584407806396, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.9474845081567764, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.12}
+
+
+  7%|▋         | 58/800 [20:43<3:19:30, 16.13s/it]
+
+
+                                                  
+
+
+
+  7%|▋         | 58/800 [20:43<3:19:30, 16.13s/it]
+
+{'loss': 0.0441, 'grad_norm': 36.219154357910156, 'learning_rate': 3.625e-05, 'num_tokens': 1172607.0, 'completions/mean_length': 19.625, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.25, 'completions/mean_terminated_length': 15.5, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.375, 'rewards/format_valid/std': 0.941858172416687, 'rewards/action_legal/mean': -0.125, 'rewards/action_legal/std': 0.33601075410842896, 'rewards/env_reward/mean': -0.4406411051750183, 'rewards/env_reward/std': 1.1344565153121948, 'rewards/belief_accuracy/mean': -0.07016775012016296, 'rewards/belief_accuracy/std': 0.057747405022382736, 'reward': -0.8589649200439453, 'reward_std': 1.3759971857070923, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.1016252487897873, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.12}
+
+
+  7%|▋         | 59/800 [21:02<3:27:18, 16.79s/it]
+
+
+                                                  
+
+
+
+  7%|▋         | 59/800 [21:02<3:27:18, 16.79s/it]
+
+
+  8%|▊         | 60/800 [21:18<3:24:02, 16.54s/it]
+
+
+                                                  
+
+
+{'loss': 0.0324, 'grad_norm': 3.5531437397003174, 'learning_rate': 3.6875e-05, 'num_tokens': 1192621.0, 'completions/mean_length': 25.4375, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.5, 'completions/mean_terminated_length': 18.875, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': -0.0625, 'rewards/format_valid/std': 1.2491933107376099, 'rewards/action_legal/mean': -0.28125, 'rewards/action_legal/std': 0.45680341124534607, 'rewards/env_reward/mean': -0.6339678764343262, 'rewards/env_reward/std': 1.5210340023040771, 'rewards/belief_accuracy/mean': -0.09130880236625671, 'rewards/belief_accuracy/std': 0.07029035687446594, 'reward': -1.242065668106079, 'reward_std': 1.7246809005737305, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.8093693852424622, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.12}
+
+
+  8%|▊         | 60/800 [21:18<3:24:02, 16.54s/it]
+
+
+  8%|▊         | 61/800 [21:36<3:30:19, 17.08s/it]
+
+
+                                                  
+
+
+{'loss': 0.0559, 'grad_norm': 5.959134578704834, 'learning_rate': 3.7500000000000003e-05, 'num_tokens': 1212032.0, 'completions/mean_length': 20.59375, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.3125, 'completions/mean_terminated_length': 15.409090995788574, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.375, 'rewards/format_valid/std': 0.941858172416687, 'rewards/action_legal/mean': -0.125, 'rewards/action_legal/std': 0.33601075410842896, 'rewards/env_reward/mean': -0.34805139899253845, 'rewards/env_reward/std': 1.100068211555481, 'rewards/belief_accuracy/mean': -0.08708333969116211, 'rewards/belief_accuracy/std': 0.06771090626716614, 'reward': -0.7708271145820618, 'reward_std': 1.4517900943756104, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.3984363079071045, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.12}
+
+
+  8%|▊         | 61/800 [21:36<3:30:19, 17.08s/it]
+
+{'loss': 0.0551, 'grad_norm': 2.832958936691284, 'learning_rate': 3.8125e-05, 'num_tokens': 1231925.0, 'completions/mean_length': 21.65625, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.25, 'completions/mean_terminated_length': 18.20833396911621, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 27.0, 'rewards/format_valid/mean': 0.5, 'rewards/format_valid/std': 0.8519427180290222, 'rewards/action_legal/mean': -0.09375, 'rewards/action_legal/std': 0.2961445748806, 'rewards/env_reward/mean': -0.02272176742553711, 'rewards/env_reward/std': 1.1089532375335693, 'rewards/belief_accuracy/mean': -0.10116032510995865, 'rewards/belief_accuracy/std': 0.07393884658813477, 'reward': -0.31725114583969116, 'reward_std': 1.4482476711273193, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.3764083683490753, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.12}
+
+
+  8%|▊         | 62/800 [21:52<3:26:05, 16.75s/it]
+
+
+                                                  
+
+
+
+  8%|▊         | 62/800 [21:52<3:26:05, 16.75s/it]
+
+
+  8%|▊         | 63/800 [22:08<3:23:00, 16.53s/it]
+
+
+                                                  
+
+
+{'loss': 0.0416, 'grad_norm': 2.552192211151123, 'learning_rate': 3.875e-05, 'num_tokens': 1251894.0, 'completions/mean_length': 24.03125, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.5, 'completions/mean_terminated_length': 16.0625, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': 0.171875, 'rewards/format_valid/std': 1.08218252658844, 'rewards/action_legal/mean': -0.1875, 'rewards/action_legal/std': 0.3965577781200409, 'rewards/env_reward/mean': -0.3593047261238098, 'rewards/env_reward/std': 1.368615984916687, 'rewards/belief_accuracy/mean': -0.11601302027702332, 'rewards/belief_accuracy/std': 0.06889842450618744, 'reward': -0.8877773880958557, 'reward_std': 1.3601653575897217, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.0387591272592545, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.13}
+
+
+  8%|▊         | 63/800 [22:08<3:23:00, 16.53s/it]
+
+{'loss': 0.0461, 'grad_norm': 6.817470550537109, 'learning_rate': 3.9375e-05, 'num_tokens': 1271686.0, 'completions/mean_length': 18.5, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.21875, 'completions/mean_terminated_length': 14.719999313354492, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': 0.6875, 'rewards/format_valid/std': 0.24593468010425568, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.2330501675605774, 'rewards/env_reward/std': 0.21573814749717712, 'rewards/belief_accuracy/mean': -0.07774622738361359, 'rewards/belief_accuracy/std': 0.09846372902393341, 'reward': 0.1507115364074707, 'reward_std': 0.21509701013565063, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.151637226343155, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.13}
+
+
+  8%|▊         | 64/800 [22:24<3:20:40, 16.36s/it]
+
+
+                                                  
+
+
+
+  8%|▊         | 64/800 [22:24<3:20:40, 16.36s/it]
+
+
+  8%|▊         | 65/800 [22:42<3:27:51, 16.97s/it]
+
+
+                                                  
+
+
+{'loss': 0.0477, 'grad_norm': 6.349569320678711, 'learning_rate': 4e-05, 'num_tokens': 1291222.0, 'completions/mean_length': 24.5, 'completions/min_length': 9.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.5625, 'completions/mean_terminated_length': 14.85714340209961, 'completions/min_terminated_length': 9.0, 'completions/max_terminated_length': 28.0, 'rewards/format_valid/mean': 0.921875, 'rewards/format_valid/std': 0.18445101380348206, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.08661454916000366, 'rewards/env_reward/std': 0.3411381244659424, 'rewards/belief_accuracy/mean': -0.08203979581594467, 'rewards/belief_accuracy/std': 0.0756392553448677, 'reward': -0.07010380923748016, 'reward_std': 0.3917754888534546, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.1919192671775818, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.13}
+
+
+  8%|▊         | 65/800 [22:42<3:27:51, 16.97s/it]
+
+
+  8%|▊         | 66/800 [22:58<3:24:07, 16.69s/it]
+
+
+                                                  
+
+
+{'loss': 0.0552, 'grad_norm': 8.538365364074707, 'learning_rate': 4.0625000000000005e-05, 'num_tokens': 1311270.0, 'completions/mean_length': 26.5, 'completions/min_length': 11.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.46875, 'completions/mean_terminated_length': 21.647058486938477, 'completions/min_terminated_length': 11.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.703125, 'rewards/format_valid/std': 0.55152827501297, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.1764737069606781, 'rewards/env_reward/std': 0.7077809572219849, 'rewards/belief_accuracy/mean': -0.09503433108329773, 'rewards/belief_accuracy/std': 0.10463222861289978, 'reward': 0.013201296329498291, 'reward_std': 0.8551115989685059, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.3804154694080353, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.13}
+
+
+  8%|▊         | 66/800 [22:58<3:24:07, 16.69s/it]
+
+{'loss': 0.0436, 'grad_norm': 18.87468719482422, 'learning_rate': 4.125e-05, 'num_tokens': 1330696.0, 'completions/mean_length': 21.0625, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.3125, 'completions/mean_terminated_length': 16.090909957885742, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.59375, 'rewards/format_valid/std': 0.8747119307518005, 'rewards/action_legal/mean': -0.09375, 'rewards/action_legal/std': 0.2961445748806, 'rewards/env_reward/mean': -0.3135613799095154, 'rewards/env_reward/std': 0.9900801181793213, 'rewards/belief_accuracy/mean': -0.06309541314840317, 'rewards/belief_accuracy/std': 0.08013322949409485, 'reward': -0.6346282958984375, 'reward_std': 0.9238852262496948, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.0900818705558777, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.13}
+
+
+  8%|▊         | 67/800 [23:17<3:29:52, 17.18s/it]
+
+
+                                                  
+
+
+
+  8%|▊         | 67/800 [23:17<3:29:52, 17.18s/it]
+
+{'loss': 0.0339, 'grad_norm': 18.394432067871094, 'learning_rate': 4.1875e-05, 'num_tokens': 1349694.0, 'completions/mean_length': 21.6875, 'completions/min_length': 9.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.4375, 'completions/mean_terminated_length': 13.666666984558105, 'completions/min_terminated_length': 9.0, 'completions/max_terminated_length': 26.0, 'rewards/format_valid/mean': 0.40625, 'rewards/format_valid/std': 1.180742621421814, 'rewards/action_legal/mean': -0.1875, 'rewards/action_legal/std': 0.3965577781200409, 'rewards/env_reward/mean': -0.1496710181236267, 'rewards/env_reward/std': 1.4231793880462646, 'rewards/belief_accuracy/mean': -0.06207277625799179, 'rewards/belief_accuracy/std': 0.08056068420410156, 'reward': -0.3997873067855835, 'reward_std': 1.4436144828796387, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.8472457826137543, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.14}
+
+
+  8%|▊         | 68/800 [23:35<3:33:44, 17.52s/it]
+
+
+                                                  
+
+
+
+  8%|▊         | 68/800 [23:35<3:33:44, 17.52s/it]
+
+{'loss': 0.0341, 'grad_norm': 13.612622261047363, 'learning_rate': 4.25e-05, 'num_tokens': 1369694.0, 'completions/mean_length': 25.0, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.5, 'completions/mean_terminated_length': 18.0, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.0625, 'rewards/format_valid/std': 1.2296733856201172, 'rewards/action_legal/mean': -0.25, 'rewards/action_legal/std': 0.4399413466453552, 'rewards/env_reward/mean': -0.6420800089836121, 'rewards/env_reward/std': 1.4343847036361694, 'rewards/belief_accuracy/mean': -0.08176907896995544, 'rewards/belief_accuracy/std': 0.07760892063379288, 'reward': -1.2178021669387817, 'reward_std': 1.2687938213348389, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.8533681333065033, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.14}
+
+
+  9%|▊         | 69/800 [23:51<3:27:39, 17.04s/it]
+
+
+                                                  
+
+
+
+  9%|▊         | 69/800 [23:51<3:27:39, 17.04s/it]
+
+
+  9%|▉         | 70/800 [24:07<3:23:30, 16.73s/it]
+
+
+                                                  
+
+
+{'loss': 0.0354, 'grad_norm': 6.78806734085083, 'learning_rate': 4.3125000000000005e-05, 'num_tokens': 1389698.0, 'completions/mean_length': 25.125, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.5, 'completions/mean_terminated_length': 18.25, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.5625, 'rewards/format_valid/std': 0.16800537705421448, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.2887897491455078, 'rewards/env_reward/std': 0.3346047103404999, 'rewards/belief_accuracy/mean': -0.09924699366092682, 'rewards/belief_accuracy/std': 0.08604496717453003, 'reward': 0.1635686308145523, 'reward_std': 0.3170303404331207, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.8842730671167374, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.14}
+
+
+  9%|▉         | 70/800 [24:07<3:23:30, 16.73s/it]
+
+
+  9%|▉         | 71/800 [24:23<3:20:31, 16.50s/it]
+
+
+                                                  
+
+
+{'loss': 0.0402, 'grad_norm': 2.555455446243286, 'learning_rate': 4.375e-05, 'num_tokens': 1409738.0, 'completions/mean_length': 26.25, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.53125, 'completions/mean_terminated_length': 19.733333587646484, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.4375, 'rewards/format_valid/std': 0.6690146923065186, 'rewards/action_legal/mean': -0.0625, 'rewards/action_legal/std': 0.24593468010425568, 'rewards/env_reward/mean': 0.004832617938518524, 'rewards/env_reward/std': 0.840225338935852, 'rewards/belief_accuracy/mean': -0.09246852993965149, 'rewards/belief_accuracy/std': 0.02612915076315403, 'reward': -0.251406729221344, 'reward_std': 1.0001490116119385, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.0042461156845093, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.14}
+
+
+  9%|▉         | 71/800 [24:23<3:20:31, 16.50s/it]
+
+{'loss': 0.034, 'grad_norm': 7.704925537109375, 'learning_rate': 4.4375e-05, 'num_tokens': 1429795.0, 'completions/mean_length': 26.78125, 'completions/min_length': 11.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.59375, 'completions/mean_terminated_length': 19.153846740722656, 'completions/min_terminated_length': 11.0, 'completions/max_terminated_length': 29.0, 'rewards/format_valid/mean': 0.375, 'rewards/format_valid/std': 0.941858172416687, 'rewards/action_legal/mean': -0.125, 'rewards/action_legal/std': 0.33601075410842896, 'rewards/env_reward/mean': -0.06938796490430832, 'rewards/env_reward/std': 1.1679229736328125, 'rewards/belief_accuracy/mean': -0.11312198638916016, 'rewards/belief_accuracy/std': 0.0759696513414383, 'reward': -0.43094784021377563, 'reward_std': 1.644728422164917, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.8508740216493607, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.14}
+
+
+  9%|▉         | 72/800 [24:39<3:18:27, 16.36s/it]
+
+
+                                                  
+
+
+
+  9%|▉         | 72/800 [24:39<3:18:27, 16.36s/it]
+
+
+  9%|▉         | 73/800 [24:55<3:16:45, 16.24s/it]
+
+
+                                                  
+
+
+{'loss': 0.0431, 'grad_norm': 24.107065200805664, 'learning_rate': 4.5e-05, 'num_tokens': 1449743.0, 'completions/mean_length': 23.375, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.5, 'completions/mean_terminated_length': 14.75, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': 0.296875, 'rewards/format_valid/std': 0.760511040687561, 'rewards/action_legal/mean': -0.09375, 'rewards/action_legal/std': 0.2961445748806, 'rewards/env_reward/mean': 0.11074212193489075, 'rewards/env_reward/std': 1.1383394002914429, 'rewards/belief_accuracy/mean': -0.10030092298984528, 'rewards/belief_accuracy/std': 0.006413234863430262, 'reward': -0.12463332712650299, 'reward_std': 1.5522685050964355, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.0765477567911148, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.15}
+
+
+  9%|▉         | 73/800 [24:55<3:16:45, 16.24s/it]
+
+{'loss': 0.0369, 'grad_norm': 5.039280414581299, 'learning_rate': 4.5625e-05, 'num_tokens': 1469730.0, 'completions/mean_length': 24.59375, 'completions/min_length': 11.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.46875, 'completions/mean_terminated_length': 18.05882453918457, 'completions/min_terminated_length': 11.0, 'completions/max_terminated_length': 29.0, 'rewards/format_valid/mean': 0.453125, 'rewards/format_valid/std': 0.9701959490776062, 'rewards/action_legal/mean': -0.125, 'rewards/action_legal/std': 0.33601075410842896, 'rewards/env_reward/mean': -0.5338205695152283, 'rewards/env_reward/std': 0.9858322739601135, 'rewards/belief_accuracy/mean': -0.08085420727729797, 'rewards/belief_accuracy/std': 0.06865385919809341, 'reward': -1.0268871784210205, 'reward_std': 0.8363633155822754, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.9230827391147614, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.15}
+
+
+  9%|▉         | 74/800 [25:11<3:15:34, 16.16s/it]
+
+
+                                                  
+
+
+
+  9%|▉         | 74/800 [25:11<3:15:34, 16.16s/it]
+
+{'loss': 0.0444, 'grad_norm': 5.296670436859131, 'learning_rate': 4.6250000000000006e-05, 'num_tokens': 1489650.0, 'completions/mean_length': 22.5, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.4375, 'completions/mean_terminated_length': 15.111111640930176, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.6875, 'rewards/format_valid/std': 0.5499266386032104, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.16421255469322205, 'rewards/env_reward/std': 0.6967208981513977, 'rewards/belief_accuracy/mean': -0.12233620882034302, 'rewards/belief_accuracy/std': 0.10031671077013016, 'reward': -0.08787727355957031, 'reward_std': 0.7855972051620483, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.1106794476509094, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.15}
+
+
+  9%|▉         | 75/800 [25:27<3:14:42, 16.11s/it]
+
+
+                                                  
+
+
+
+  9%|▉         | 75/800 [25:27<3:14:42, 16.11s/it]
+
+{'loss': 0.0411, 'grad_norm': 2.3795509338378906, 'learning_rate': 4.6875e-05, 'num_tokens': 1509635.0, 'completions/mean_length': 24.53125, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.46875, 'completions/mean_terminated_length': 17.941177368164062, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 29.0, 'rewards/format_valid/mean': 0.671875, 'rewards/format_valid/std': 0.5478605628013611, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.0663258284330368, 'rewards/env_reward/std': 0.6256977319717407, 'rewards/belief_accuracy/mean': -0.11885273456573486, 'rewards/belief_accuracy/std': 0.048995744436979294, 'reward': -0.22503823041915894, 'reward_std': 0.5947009325027466, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.026912271976471, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.15}
+
+
+ 10%|▉         | 76/800 [25:43<3:13:59, 16.08s/it]
+
+
+                                                  
+
+
+
+ 10%|▉         | 76/800 [25:43<3:13:59, 16.08s/it]
+
+
+ 10%|▉         | 77/800 [26:01<3:21:44, 16.74s/it]
+
+
+                                                  
+
+
+{'loss': 0.0523, 'grad_norm': 3.201648473739624, 'learning_rate': 4.75e-05, 'num_tokens': 1528954.0, 'completions/mean_length': 17.71875, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.1875, 'completions/mean_terminated_length': 14.423077583312988, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': 0.703125, 'rewards/format_valid/std': 0.7389985918998718, 'rewards/action_legal/mean': -0.0625, 'rewards/action_legal/std': 0.24593468010425568, 'rewards/env_reward/mean': -0.07831752300262451, 'rewards/env_reward/std': 0.8618139624595642, 'rewards/belief_accuracy/mean': -0.0181246530264616, 'rewards/belief_accuracy/std': 0.1005098894238472, 'reward': -0.1398189663887024, 'reward_std': 1.1272202730178833, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.3069145679473877, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.15}
+
+
+ 10%|▉         | 77/800 [26:01<3:21:44, 16.74s/it]
+
+
+ 10%|▉         | 78/800 [26:17<3:18:47, 16.52s/it]
+
+
+                                                  
+
+
+{'loss': 0.0421, 'grad_norm': 2.4587528705596924, 'learning_rate': 4.8125000000000004e-05, 'num_tokens': 1548851.0, 'completions/mean_length': 21.78125, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.28125, 'completions/mean_terminated_length': 17.782609939575195, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': 0.203125, 'rewards/format_valid/std': 0.9825852513313293, 'rewards/action_legal/mean': -0.15625, 'rewards/action_legal/std': 0.3689020276069641, 'rewards/env_reward/mean': -0.4690311551094055, 'rewards/env_reward/std': 1.1214438676834106, 'rewards/belief_accuracy/mean': -0.11088056862354279, 'rewards/belief_accuracy/std': 0.03750878944993019, 'reward': -1.0338447093963623, 'reward_std': 1.2146897315979004, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.0524000227451324, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.16}
+
+
+ 10%|▉         | 78/800 [26:17<3:18:47, 16.52s/it]
+
+{'loss': 0.037, 'grad_norm': 2.1357271671295166, 'learning_rate': 4.875e-05, 'num_tokens': 1568822.0, 'completions/mean_length': 24.09375, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.375, 'completions/mean_terminated_length': 19.350000381469727, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.65625, 'rewards/format_valid/std': 0.23546454310417175, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.04080890491604805, 'rewards/env_reward/std': 0.39410626888275146, 'rewards/belief_accuracy/mean': -0.07873205840587616, 'rewards/belief_accuracy/std': 0.053090181201696396, 'reward': -0.14217031002044678, 'reward_std': 0.32530638575553894, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.9245329350233078, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.16}
+
+
+ 10%|▉         | 79/800 [26:33<3:16:44, 16.37s/it]
+
+
+                                                  
+
+
+
+ 10%|▉         | 79/800 [26:33<3:16:44, 16.37s/it]
+
+
+ 10%|█         | 80/800 [26:49<3:14:58, 16.25s/it]
+
+
+                                                  
+
+
+{'loss': 0.0518, 'grad_norm': 3.7176260948181152, 'learning_rate': 4.937500000000001e-05, 'num_tokens': 1588733.0, 'completions/mean_length': 22.21875, 'completions/min_length': 10.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.28125, 'completions/mean_terminated_length': 18.39130401611328, 'completions/min_terminated_length': 10.0, 'completions/max_terminated_length': 28.0, 'rewards/format_valid/mean': 0.546875, 'rewards/format_valid/std': 0.5136739015579224, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.1239052414894104, 'rewards/env_reward/std': 0.6572784185409546, 'rewards/belief_accuracy/mean': -0.09458260238170624, 'rewards/belief_accuracy/std': 0.03678236901760101, 'reward': -0.0721086859703064, 'reward_std': 0.69075608253479, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.2951692044734955, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.16}
+
+
+ 10%|█         | 80/800 [26:49<3:14:58, 16.25s/it]
+
+
+ 10%|█         | 81/800 [27:05<3:13:35, 16.16s/it]
+
+
+                                                  
+
+
+{'loss': 0.0324, 'grad_norm': 12.331483840942383, 'learning_rate': 5e-05, 'num_tokens': 1608737.0, 'completions/mean_length': 25.125, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.53125, 'completions/mean_terminated_length': 17.33333396911621, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 28.0, 'rewards/format_valid/mean': 0.609375, 'rewards/format_valid/std': 0.2100067138671875, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.06090973690152168, 'rewards/env_reward/std': 0.34690457582473755, 'rewards/belief_accuracy/mean': -0.08593874424695969, 'rewards/belief_accuracy/std': 0.04739077389240265, 'reward': -0.13598290085792542, 'reward_std': 0.35032451152801514, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.8107591271400452, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.16}
+
+
+ 10%|█         | 81/800 [27:05<3:13:35, 16.16s/it]
+
+{'loss': 0.0487, 'grad_norm': 3.0294620990753174, 'learning_rate': 4.993055555555556e-05, 'num_tokens': 1628635.0, 'completions/mean_length': 21.8125, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.375, 'completions/mean_terminated_length': 15.699999809265137, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': 0.796875, 'rewards/format_valid/std': 0.24949544668197632, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.20791909098625183, 'rewards/env_reward/std': 0.3105705976486206, 'rewards/belief_accuracy/mean': -0.12021970748901367, 'rewards/belief_accuracy/std': 0.11214939504861832, 'reward': -0.008936740458011627, 'reward_std': 0.3372165560722351, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.2180950939655304, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.16}
+
+
+ 10%|█         | 82/800 [27:21<3:12:36, 16.10s/it]
+
+
+                                                  
+
+
+
+ 10%|█         | 82/800 [27:21<3:12:36, 16.10s/it]
+
+{'loss': 0.0597, 'grad_norm': 3.1383140087127686, 'learning_rate': 4.986111111111111e-05, 'num_tokens': 1648544.0, 'completions/mean_length': 22.15625, 'completions/min_length': 9.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.40625, 'completions/mean_terminated_length': 15.421052932739258, 'completions/min_terminated_length': 9.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.671875, 'rewards/format_valid/std': 0.5478605628013611, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': -0.018333418294787407, 'rewards/env_reward/std': 0.6250711679458618, 'rewards/belief_accuracy/mean': -0.148574098944664, 'rewards/belief_accuracy/std': 0.08691953122615814, 'reward': -0.4411911964416504, 'reward_std': 0.6577557921409607, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.4920355677604675, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.17}
+
+
+ 10%|█         | 83/800 [27:37<3:11:54, 16.06s/it]
+
+
+                                                  
+
+
+
+ 10%|█         | 83/800 [27:37<3:11:54, 16.06s/it]
+
+
+ 10%|█         | 84/800 [27:53<3:11:37, 16.06s/it]
+
+
+                                                  
+
+
+{'loss': 0.028, 'grad_norm': 1.8351413011550903, 'learning_rate': 4.979166666666667e-05, 'num_tokens': 1668673.0, 'completions/mean_length': 29.03125, 'completions/min_length': 17.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.65625, 'completions/mean_terminated_length': 23.363637924194336, 'completions/min_terminated_length': 17.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.734375, 'rewards/format_valid/std': 0.5533530116081238, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.054470811039209366, 'rewards/env_reward/std': 0.6041941046714783, 'rewards/belief_accuracy/mean': -0.0948885828256607, 'rewards/belief_accuracy/std': 0.046606212854385376, 'reward': -0.16780328750610352, 'reward_std': 0.5828790068626404, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.7002743929624557, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.17}
+
+
+ 10%|█         | 84/800 [27:53<3:11:37, 16.06s/it]
+
+{'loss': 0.0376, 'grad_norm': 2.3608760833740234, 'learning_rate': 4.972222222222223e-05, 'num_tokens': 1688760.0, 'completions/mean_length': 27.71875, 'completions/min_length': 11.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.65625, 'completions/mean_terminated_length': 19.545455932617188, 'completions/min_terminated_length': 11.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.640625, 'rewards/format_valid/std': 0.8820282220840454, 'rewards/action_legal/mean': -0.09375, 'rewards/action_legal/std': 0.2961445748806, 'rewards/env_reward/mean': -0.2076132595539093, 'rewards/env_reward/std': 0.946779191493988, 'rewards/belief_accuracy/mean': -0.07281694561243057, 'rewards/belief_accuracy/std': 0.1268777847290039, 'reward': -0.5025269389152527, 'reward_std': 1.3361655473709106, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.9394793063402176, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.17}
+
+
+ 11%|█         | 85/800 [28:09<3:11:06, 16.04s/it]
+
+
+                                                  
+
+
+
+ 11%|█         | 85/800 [28:09<3:11:06, 16.04s/it]
+
+{'loss': 0.0401, 'grad_norm': 1.812410831451416, 'learning_rate': 4.965277777777778e-05, 'num_tokens': 1708846.0, 'completions/mean_length': 27.6875, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.71875, 'completions/mean_terminated_length': 16.66666603088379, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.515625, 'rewards/format_valid/std': 0.8565155863761902, 'rewards/action_legal/mean': -0.09375, 'rewards/action_legal/std': 0.2961445748806, 'rewards/env_reward/mean': -0.04423265904188156, 'rewards/env_reward/std': 1.0034382343292236, 'rewards/belief_accuracy/mean': -0.10080601274967194, 'rewards/belief_accuracy/std': 0.06879791617393494, 'reward': -0.347673237323761, 'reward_std': 1.208042860031128, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.0034183412790298, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.17}
+
+
+ 11%|█         | 86/800 [28:25<3:10:39, 16.02s/it]
+
+
+                                                  
+
+
+
+ 11%|█         | 86/800 [28:25<3:10:39, 16.02s/it]
+
+
+ 11%|█         | 87/800 [28:41<3:10:26, 16.03s/it]
+
+
+                                                  
+
+
+{'loss': 0.0669, 'grad_norm': 3.7876245975494385, 'learning_rate': 4.958333333333334e-05, 'num_tokens': 1728847.0, 'completions/mean_length': 25.03125, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.59375, 'completions/mean_terminated_length': 14.84615421295166, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 28.0, 'rewards/format_valid/mean': 0.78125, 'rewards/format_valid/std': 0.2520080506801605, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.16712062060832977, 'rewards/env_reward/std': 0.18340182304382324, 'rewards/belief_accuracy/mean': -0.10623374581336975, 'rewards/belief_accuracy/std': 0.09267499297857285, 'reward': -0.02895783632993698, 'reward_std': 0.26314201951026917, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.671981394290924, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.17}
+
+
+ 11%|█         | 87/800 [28:41<3:10:26, 16.03s/it]
+
+{'loss': 0.035, 'grad_norm': 10.536504745483398, 'learning_rate': 4.951388888888889e-05, 'num_tokens': 1749010.0, 'completions/mean_length': 30.09375, 'completions/min_length': 20.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.6875, 'completions/mean_terminated_length': 25.899999618530273, 'completions/min_terminated_length': 20.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.796875, 'rewards/format_valid/std': 0.55152827501297, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.20009490847587585, 'rewards/env_reward/std': 0.6622443199157715, 'rewards/belief_accuracy/mean': -0.06206507980823517, 'rewards/belief_accuracy/std': 0.07661604136228561, 'reward': 0.15222835540771484, 'reward_std': 0.7593594193458557, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.8747647255659103, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.18}
+
+
+ 11%|█         | 88/800 [28:57<3:10:16, 16.04s/it]
+
+
+                                                  
+
+
+
+ 11%|█         | 88/800 [28:57<3:10:16, 16.04s/it]
+
+
+ 11%|█         | 89/800 [29:13<3:09:50, 16.02s/it]
+
+
+                                                  
+
+
+{'loss': 0.0269, 'grad_norm': 3.0154426097869873, 'learning_rate': 4.9444444444444446e-05, 'num_tokens': 1769172.0, 'completions/mean_length': 30.0625, 'completions/min_length': 17.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.75, 'completions/mean_terminated_length': 24.25, 'completions/min_terminated_length': 17.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': 0.53125, 'rewards/format_valid/std': 0.9913944005966187, 'rewards/action_legal/mean': -0.125, 'rewards/action_legal/std': 0.33601075410842896, 'rewards/env_reward/mean': -0.06829120963811874, 'rewards/env_reward/std': 1.1849027872085571, 'rewards/belief_accuracy/mean': -0.10869339108467102, 'rewards/belief_accuracy/std': 0.06799063086509705, 'reward': -0.40820446610450745, 'reward_std': 1.5518081188201904, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.6722820773720741, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.18}
+
+
+ 11%|█         | 89/800 [29:13<3:09:50, 16.02s/it]
+
+{'loss': 0.0325, 'grad_norm': 1.795458197593689, 'learning_rate': 4.937500000000001e-05, 'num_tokens': 1789312.0, 'completions/mean_length': 29.375, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.71875, 'completions/mean_terminated_length': 22.66666603088379, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.53125, 'rewards/format_valid/std': 0.8607714176177979, 'rewards/action_legal/mean': -0.09375, 'rewards/action_legal/std': 0.2961445748806, 'rewards/env_reward/mean': -0.16465601325035095, 'rewards/env_reward/std': 0.9621924757957458, 'rewards/belief_accuracy/mean': -0.09815632551908493, 'rewards/belief_accuracy/std': 0.06358767300844193, 'reward': -0.5195779800415039, 'reward_std': 1.2243459224700928, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.8120120018720627, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.18}
+
+
+ 11%|█▏        | 90/800 [29:29<3:09:20, 16.00s/it]
+
+
+                                                  
+
+
+
+ 11%|█▏        | 90/800 [29:29<3:09:20, 16.00s/it]
+
+
+ 11%|█▏        | 91/800 [29:45<3:09:06, 16.00s/it]
+
+
+                                                  
+
+
+{'loss': 0.0259, 'grad_norm': 3.676872491836548, 'learning_rate': 4.930555555555556e-05, 'num_tokens': 1809500.0, 'completions/mean_length': 30.875, 'completions/min_length': 19.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.875, 'completions/mean_terminated_length': 23.0, 'completions/min_terminated_length': 19.0, 'completions/max_terminated_length': 26.0, 'rewards/format_valid/mean': 0.5625, 'rewards/format_valid/std': 0.9979817867279053, 'rewards/action_legal/mean': -0.125, 'rewards/action_legal/std': 0.33601075410842896, 'rewards/env_reward/mean': -0.041041985154151917, 'rewards/env_reward/std': 1.1760306358337402, 'rewards/belief_accuracy/mean': -0.03627660125494003, 'rewards/belief_accuracy/std': 0.08266113698482513, 'reward': -0.14851778745651245, 'reward_std': 1.4743738174438477, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.6479260548949242, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.18}
+
+
+ 11%|█▏        | 91/800 [29:45<3:09:06, 16.00s/it]
+
+{'loss': 0.0375, 'grad_norm': 2.2367706298828125, 'learning_rate': 4.923611111111112e-05, 'num_tokens': 1829555.0, 'completions/mean_length': 26.71875, 'completions/min_length': 9.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.6875, 'completions/mean_terminated_length': 15.100000381469727, 'completions/min_terminated_length': 9.0, 'completions/max_terminated_length': 27.0, 'rewards/format_valid/mean': 0.53125, 'rewards/format_valid/std': 1.1067060232162476, 'rewards/action_legal/mean': -0.15625, 'rewards/action_legal/std': 0.3689020276069641, 'rewards/env_reward/mean': -0.26690298318862915, 'rewards/env_reward/std': 1.2277095317840576, 'rewards/belief_accuracy/mean': -0.12818560004234314, 'rewards/belief_accuracy/std': 0.07785481959581375, 'reward': -0.7661612033843994, 'reward_std': 1.217362403869629, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.9375400245189667, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.18}
+
+
+ 12%|█▏        | 92/800 [30:01<3:08:55, 16.01s/it]
+
+
+                                                  
+
+
+
+ 12%|█▏        | 92/800 [30:01<3:08:55, 16.01s/it]
+
+{'loss': 0.0275, 'grad_norm': 1.945510983467102, 'learning_rate': 4.9166666666666665e-05, 'num_tokens': 1849736.0, 'completions/mean_length': 30.65625, 'completions/min_length': 23.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.8125, 'completions/mean_terminated_length': 24.83333396911621, 'completions/min_terminated_length': 23.0, 'completions/max_terminated_length': 27.0, 'rewards/format_valid/mean': 0.890625, 'rewards/format_valid/std': 0.2100067138671875, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.36152032017707825, 'rewards/env_reward/std': 0.38280951976776123, 'rewards/belief_accuracy/mean': -0.062480323016643524, 'rewards/belief_accuracy/std': 0.07967144250869751, 'reward': 0.39937078952789307, 'reward_std': 0.5459688901901245, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.6881065517663956, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.19}
+
+
+ 12%|█▏        | 93/800 [30:17<3:08:33, 16.00s/it]
+
+
+                                                  
+
+
+
+ 12%|█▏        | 93/800 [30:17<3:08:33, 16.00s/it]
+
+
+ 12%|█▏        | 94/800 [30:33<3:08:08, 15.99s/it]
+
+
+                                                  
+
+
+{'loss': 0.0258, 'grad_norm': 1.7560251951217651, 'learning_rate': 4.909722222222222e-05, 'num_tokens': 1869896.0, 'completions/mean_length': 30.0, 'completions/min_length': 11.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.8125, 'completions/mean_terminated_length': 21.33333396911621, 'completions/min_terminated_length': 11.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': 0.765625, 'rewards/format_valid/std': 0.253503680229187, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.40401214361190796, 'rewards/env_reward/std': 0.24956835806369781, 'rewards/belief_accuracy/mean': -0.06804037094116211, 'rewards/belief_accuracy/std': 0.07072039693593979, 'reward': 0.4401783347129822, 'reward_std': 0.37832456827163696, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.6456379219889641, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.19}
+
+
+ 12%|█▏        | 94/800 [30:33<3:08:08, 15.99s/it]
+
+{'loss': 0.0369, 'grad_norm': 2.5607857704162598, 'learning_rate': 4.902777777777778e-05, 'num_tokens': 1890022.0, 'completions/mean_length': 28.9375, 'completions/min_length': 15.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.65625, 'completions/mean_terminated_length': 23.090909957885742, 'completions/min_terminated_length': 15.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.671875, 'rewards/format_valid/std': 0.7362653613090515, 'rewards/action_legal/mean': -0.0625, 'rewards/action_legal/std': 0.24593468010425568, 'rewards/env_reward/mean': 0.22164197266101837, 'rewards/env_reward/std': 0.8898347616195679, 'rewards/belief_accuracy/mean': -0.07199910283088684, 'rewards/belief_accuracy/std': 0.07257580757141113, 'reward': 0.1469343900680542, 'reward_std': 0.9647181034088135, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.9233426451683044, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.19}
+
+
+ 12%|█▏        | 95/800 [30:49<3:07:53, 15.99s/it]
+
+
+                                                  
+
+
+
+ 12%|█▏        | 95/800 [30:49<3:07:53, 15.99s/it]
+
+{'loss': 0.0266, 'grad_norm': 3.2734971046447754, 'learning_rate': 4.8958333333333335e-05, 'num_tokens': 1910217.0, 'completions/mean_length': 31.09375, 'completions/min_length': 18.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.875, 'completions/mean_terminated_length': 24.75, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.859375, 'rewards/format_valid/std': 0.22840170562267303, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.4473568797111511, 'rewards/env_reward/std': 0.3304316997528076, 'rewards/belief_accuracy/mean': -0.08798785507678986, 'rewards/belief_accuracy/std': 0.06342029571533203, 'reward': 0.4500405192375183, 'reward_std': 0.30363211035728455, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.6640048772096634, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.19}
+
+
+ 12%|█▏        | 96/800 [31:05<3:07:38, 15.99s/it]
+
+
+                                                  
+
+
+
+ 12%|█▏        | 96/800 [31:05<3:07:38, 15.99s/it]
+
+
+ 12%|█▏        | 97/800 [31:21<3:07:34, 16.01s/it]
+
+
+                                                  
+
+
+{'loss': 0.0347, 'grad_norm': 2.3364200592041016, 'learning_rate': 4.888888888888889e-05, 'num_tokens': 1930325.0, 'completions/mean_length': 28.375, 'completions/min_length': 11.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.59375, 'completions/mean_terminated_length': 23.076923370361328, 'completions/min_terminated_length': 11.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.796875, 'rewards/format_valid/std': 0.55152827501297, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': -0.2400549054145813, 'rewards/env_reward/std': 0.583495557308197, 'rewards/belief_accuracy/mean': -0.08408010005950928, 'rewards/belief_accuracy/std': 0.06422320008277893, 'reward': -0.5740413665771484, 'reward_std': 0.6673891544342041, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.8677158802747726, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.19}
+
+
+ 12%|█▏        | 97/800 [31:21<3:07:34, 16.01s/it]
+
+{'loss': 0.054, 'grad_norm': 1.5626296997070312, 'learning_rate': 4.8819444444444444e-05, 'num_tokens': 1950224.0, 'completions/mean_length': 21.84375, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.5, 'completions/mean_terminated_length': 11.6875, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 24.0, 'rewards/format_valid/mean': 0.578125, 'rewards/format_valid/std': 0.7196479439735413, 'rewards/action_legal/mean': -0.0625, 'rewards/action_legal/std': 0.24593468010425568, 'rewards/env_reward/mean': -0.18073144555091858, 'rewards/env_reward/std': 0.8218822479248047, 'rewards/belief_accuracy/mean': -0.07426592707633972, 'rewards/belief_accuracy/std': 0.04626186192035675, 'reward': -0.46811366081237793, 'reward_std': 1.171541452407837, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.3493715822696686, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.2}
+
+
+ 12%|█▏        | 98/800 [31:37<3:07:10, 16.00s/it]
+
+
+                                                  
+
+
+
+ 12%|█▏        | 98/800 [31:37<3:07:10, 16.00s/it]
+
+{'loss': 0.039, 'grad_norm': 1.581009030342102, 'learning_rate': 4.875e-05, 'num_tokens': 1970223.0, 'completions/mean_length': 24.96875, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.59375, 'completions/mean_terminated_length': 14.69230842590332, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.71875, 'rewards/format_valid/std': 0.5526694655418396, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.2580660581588745, 'rewards/env_reward/std': 0.7565141320228577, 'rewards/belief_accuracy/mean': -0.04886733368039131, 'rewards/belief_accuracy/std': 0.08882594853639603, 'reward': 0.27487215399742126, 'reward_std': 0.7028729915618896, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.9750550836324692, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.2}
+
+
+ 12%|█▏        | 99/800 [31:53<3:06:47, 15.99s/it]
+
+
+                                                  
+
+
+
+ 12%|█▏        | 99/800 [31:53<3:06:47, 15.99s/it]
+
+
+ 12%|█▎        | 100/800 [32:09<3:06:34, 15.99s/it]
+
+
+                                                   
+
+
+{'loss': 0.0444, 'grad_norm': 1.8844627141952515, 'learning_rate': 4.8680555555555554e-05, 'num_tokens': 1990212.0, 'completions/mean_length': 24.65625, 'completions/min_length': 11.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.5, 'completions/mean_terminated_length': 17.3125, 'completions/min_terminated_length': 11.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.6875, 'rewards/format_valid/std': 0.7378040552139282, 'rewards/action_legal/mean': -0.0625, 'rewards/action_legal/std': 0.24593468010425568, 'rewards/env_reward/mean': 0.117036372423172, 'rewards/env_reward/std': 0.9240343570709229, 'rewards/belief_accuracy/mean': -0.08031107485294342, 'rewards/belief_accuracy/std': 0.0689208060503006, 'reward': -0.03412869945168495, 'reward_std': 1.1884723901748657, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.1093961596488953, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.2}
+
+
+ 12%|█▎        | 100/800 [32:09<3:06:34, 15.99s/it]
+
+
+ 13%|█▎        | 101/800 [32:25<3:06:26, 16.00s/it]
+
+
+                                                   
+
+
+{'loss': 0.0597, 'grad_norm': 1.6261900663375854, 'learning_rate': 4.8611111111111115e-05, 'num_tokens': 2010137.0, 'completions/mean_length': 22.65625, 'completions/min_length': 5.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.3125, 'completions/mean_terminated_length': 18.40909194946289, 'completions/min_terminated_length': 5.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.71875, 'rewards/format_valid/std': 0.2520080506801605, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.3402489125728607, 'rewards/env_reward/std': 0.2843996286392212, 'rewards/belief_accuracy/mean': -0.10034549236297607, 'rewards/belief_accuracy/std': 0.04586603492498398, 'reward': 0.24527442455291748, 'reward_std': 0.17142976820468903, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.4926097691059113, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.2}
+
+
+ 13%|█▎        | 101/800 [32:25<3:06:26, 16.00s/it]
+
+{'loss': 0.0537, 'grad_norm': 1.3859573602676392, 'learning_rate': 4.854166666666667e-05, 'num_tokens': 2030000.0, 'completions/mean_length': 20.71875, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.28125, 'completions/mean_terminated_length': 16.30434799194336, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 26.0, 'rewards/format_valid/mean': 0.671875, 'rewards/format_valid/std': 0.24127934873104095, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.42291414737701416, 'rewards/env_reward/std': 0.4262447953224182, 'rewards/belief_accuracy/mean': -0.09606651216745377, 'rewards/belief_accuracy/std': 0.03892457112669945, 'reward': 0.3797653913497925, 'reward_std': 0.24856865406036377, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.3422937393188477, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.2}
+
+
+ 13%|█▎        | 102/800 [32:41<3:06:19, 16.02s/it]
+
+
+                                                   
+
+
+
+ 13%|█▎        | 102/800 [32:41<3:06:19, 16.02s/it]
+
+
+ 13%|█▎        | 103/800 [32:57<3:05:57, 16.01s/it]
+
+{'loss': 0.0741, 'grad_norm': 1.5341322422027588, 'learning_rate': 4.8472222222222224e-05, 'num_tokens': 2049697.0, 'completions/mean_length': 15.53125, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.125, 'completions/mean_terminated_length': 13.178571701049805, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 29.0, 'rewards/format_valid/mean': 0.515625, 'rewards/format_valid/std': 0.4997479021549225, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.09088720381259918, 'rewards/env_reward/std': 0.5832324624061584, 'rewards/belief_accuracy/mean': -0.0909409448504448, 'rewards/belief_accuracy/std': 0.0288107730448246, 'reward': -0.11227327585220337, 'reward_std': 0.5574772357940674, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.8524042069911957, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.21}
+
+
+                                                   
+
+
+
+ 13%|█▎        | 103/800 [32:57<3:05:57, 16.01s/it]
+
+
+ 13%|█▎        | 104/800 [33:16<3:14:07, 16.74s/it]
+
+
+                                                   
+
+
+{'loss': 0.05, 'grad_norm': 1.5351487398147583, 'learning_rate': 4.840277777777778e-05, 'num_tokens': 2069099.0, 'completions/mean_length': 20.3125, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.375, 'completions/mean_terminated_length': 13.300000190734863, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': 0.703125, 'rewards/format_valid/std': 0.24949544668197632, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.3505854308605194, 'rewards/env_reward/std': 0.2871858775615692, 'rewards/belief_accuracy/mean': -0.0695766806602478, 'rewards/belief_accuracy/std': 0.047568179666996, 'reward': 0.3523043394088745, 'reward_std': 0.2911291718482971, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.2502709329128265, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.21}
+
+
+ 13%|█▎        | 104/800 [33:16<3:14:07, 16.74s/it]
+
+
+ 13%|█▎        | 105/800 [33:32<3:11:20, 16.52s/it]
+
+
+                                                   
+
+
+{'loss': 0.0676, 'grad_norm': 1.772566795349121, 'learning_rate': 4.8333333333333334e-05, 'num_tokens': 2088906.0, 'completions/mean_length': 18.96875, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.34375, 'completions/mean_terminated_length': 12.142857551574707, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 24.0, 'rewards/format_valid/mean': 0.546875, 'rewards/format_valid/std': 0.5136739015579224, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.36799654364585876, 'rewards/env_reward/std': 0.7765555381774902, 'rewards/belief_accuracy/mean': -0.10105323791503906, 'rewards/belief_accuracy/std': 0.034281887114048004, 'reward': 0.2746163606643677, 'reward_std': 0.7606244087219238, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.6895664632320404, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.21}
+
+
+ 13%|█▎        | 105/800 [33:32<3:11:20, 16.52s/it]
+
+
+ 13%|█▎        | 106/800 [36:12<11:31:21, 59.77s/it]
+
+
+                                                    
+
+
+{'loss': 0.0656, 'grad_norm': 0.9811370968818665, 'learning_rate': 4.8263888888888895e-05, 'num_tokens': 2108515.0, 'completions/mean_length': 12.78125, 'completions/min_length': 7.0, 'completions/max_length': 29.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 12.78125, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 29.0, 'rewards/format_valid/mean': 0.734375, 'rewards/format_valid/std': 0.253503680229187, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.023011382669210434, 'rewards/env_reward/std': 0.30233117938041687, 'rewards/belief_accuracy/mean': -0.08731750398874283, 'rewards/belief_accuracy/std': 0.04701146483421326, 'reward': -0.1907166689634323, 'reward_std': 0.13857102394104004, 'frac_reward_zero_std': 0.25, 'completion_length': 29.0, 'kl': 1.6408880949020386, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.21}
+
+
+ 13%|█▎        | 106/800 [36:12<11:31:21, 59.77s/it]
+
+{'loss': 0.0783, 'grad_norm': 1.2224558591842651, 'learning_rate': 4.819444444444445e-05, 'num_tokens': 2127701.0, 'completions/mean_length': 13.5625, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.0625, 'completions/mean_terminated_length': 12.333333969116211, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': 0.71875, 'rewards/format_valid/std': 0.2520080506801605, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.1592312455177307, 'rewards/env_reward/std': 0.2973494231700897, 'rewards/belief_accuracy/mean': -0.07070688158273697, 'rewards/belief_accuracy/std': 0.04900214076042175, 'reward': 0.0626637190580368, 'reward_std': 0.3619482219219208, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.9578028917312622, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.21}
+
+
+ 13%|█▎        | 107/800 [36:31<9:06:47, 47.34s/it] 
+
+
+                                                   
+
+
+
+ 13%|█▎        | 107/800 [36:31<9:06:47, 47.34s/it]
+
+
+ 14%|█▎        | 108/800 [36:47<7:17:27, 37.93s/it]
+
+
+                                                   
+
+
+{'loss': 0.081, 'grad_norm': 1.1701925992965698, 'learning_rate': 4.8125000000000004e-05, 'num_tokens': 2147308.0, 'completions/mean_length': 12.71875, 'completions/min_length': 5.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.0625, 'completions/mean_terminated_length': 11.433334350585938, 'completions/min_terminated_length': 5.0, 'completions/max_terminated_length': 25.0, 'rewards/format_valid/mean': 0.578125, 'rewards/format_valid/std': 0.18445101380348206, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.23115666210651398, 'rewards/env_reward/std': 0.21159517765045166, 'rewards/belief_accuracy/mean': -0.09911426156759262, 'rewards/belief_accuracy/std': 0.024266300722956657, 'reward': 0.07829844206571579, 'reward_std': 0.20249176025390625, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 2.026209771633148, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.22}
+
+
+ 14%|█▎        | 108/800 [36:47<7:17:27, 37.93s/it]
+
+{'loss': 0.0704, 'grad_norm': 1.3266533613204956, 'learning_rate': 4.805555555555556e-05, 'num_tokens': 2167028.0, 'completions/mean_length': 16.25, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.1875, 'completions/mean_terminated_length': 12.615385055541992, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 22.0, 'rewards/format_valid/mean': 0.5625, 'rewards/format_valid/std': 0.16800537705421448, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.31564077734947205, 'rewards/env_reward/std': 0.2717551589012146, 'rewards/belief_accuracy/mean': -0.09910529851913452, 'rewards/belief_accuracy/std': 0.025613538920879364, 'reward': 0.20427027344703674, 'reward_std': 0.27431046962738037, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.759486049413681, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.22}
+
+
+ 14%|█▎        | 109/800 [37:03<6:00:56, 31.34s/it]
+
+
+                                                   
+
+
+{'loss': 0.0696, 'grad_norm': 1.4074373245239258, 'learning_rate': 4.7986111111111113e-05, 'num_tokens': 2186772.0, 'completions/mean_length': 17.0, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.125, 'completions/mean_terminated_length': 14.85714340209961, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': 0.3125, 'rewards/format_valid/std': 0.7698973417282104, 'rewards/action_legal/mean': -0.09375, 'rewards/action_legal/std': 0.2961445748806, 'rewards/env_reward/mean': -0.10502271354198456, 'rewards/env_reward/std': 1.0848227739334106, 'rewards/belief_accuracy/mean': -0.09730367362499237, 'rewards/belief_accuracy/std': 0.018088219687342644, 'reward': -0.438507616519928, 'reward_std': 1.2971855401992798, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.7397008538246155, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.22}
+
+
+ 14%|█▎        | 109/800 [37:03<6:00:56, 31.34s/it]
+
+
+ 14%|█▍        | 110/800 [37:19<5:07:18, 26.72s/it]
+
+
+                                                   
+
+
+
+ 14%|█▍        | 110/800 [37:19<5:07:18, 26.72s/it]
+
+
+ 14%|█▍        | 111/800 [37:35<4:29:45, 23.49s/it]
+
+
+                                                   
+
+
+{'loss': 0.0589, 'grad_norm': 1.0852833986282349, 'learning_rate': 4.791666666666667e-05, 'num_tokens': 2206496.0, 'completions/mean_length': 16.375, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.21875, 'completions/mean_terminated_length': 12.0, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 22.0, 'rewards/format_valid/mean': 0.46875, 'rewards/format_valid/std': 0.4741290807723999, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.19204296171665192, 'rewards/env_reward/std': 0.6440114378929138, 'rewards/belief_accuracy/mean': -0.09696885198354721, 'rewards/belief_accuracy/std': 0.013144799508154392, 'reward': 0.019032884389162064, 'reward_std': 0.78080153465271, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.473556399345398, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.22}
+
+
+ 14%|█▍        | 111/800 [37:35<4:29:45, 23.49s/it]
+
+
+ 14%|█▍        | 112/800 [37:51<4:03:35, 21.24s/it]
+
+
+                                                   
+
+
+{'loss': 0.0707, 'grad_norm': 1.1323822736740112, 'learning_rate': 4.784722222222223e-05, 'num_tokens': 2226240.0, 'completions/mean_length': 17.0, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.1875, 'completions/mean_terminated_length': 13.538461685180664, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 28.0, 'rewards/format_valid/mean': 0.390625, 'rewards/format_valid/std': 0.6442596912384033, 'rewards/action_legal/mean': -0.0625, 'rewards/action_legal/std': 0.24593468010425568, 'rewards/env_reward/mean': 0.19269713759422302, 'rewards/env_reward/std': 0.9365795850753784, 'rewards/belief_accuracy/mean': -0.09713741391897202, 'rewards/belief_accuracy/std': 0.030002161860466003, 'reward': 0.014039717614650726, 'reward_std': 0.9618839621543884, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.7663409113883972, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.22}
+
+
+ 14%|█▍        | 112/800 [37:51<4:03:35, 21.24s/it]
+
+
+ 14%|█▍        | 113/800 [38:07<3:45:08, 19.66s/it]
+
+
+                                                   
+
+
+{'loss': 0.0658, 'grad_norm': 1.3195735216140747, 'learning_rate': 4.7777777777777784e-05, 'num_tokens': 2245980.0, 'completions/mean_length': 16.875, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.21875, 'completions/mean_terminated_length': 12.639999389648438, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.359375, 'rewards/format_valid/std': 0.6252015829086304, 'rewards/action_legal/mean': -0.0625, 'rewards/action_legal/std': 0.24593468010425568, 'rewards/env_reward/mean': 0.2983209788799286, 'rewards/env_reward/std': 1.007895827293396, 'rewards/belief_accuracy/mean': -0.10092592239379883, 'rewards/belief_accuracy/std': 0.005237827077507973, 'reward': 0.15954747796058655, 'reward_std': 1.2675752639770508, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.6461110413074493, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.23}
+
+
+ 14%|█▍        | 113/800 [38:07<3:45:08, 19.66s/it]
+
+
+ 14%|█▍        | 114/800 [38:23<3:32:21, 18.57s/it]
+
+
+                                                   
+
+
+{'loss': 0.0634, 'grad_norm': 2.070110559463501, 'learning_rate': 4.770833333333334e-05, 'num_tokens': 2265662.0, 'completions/mean_length': 15.0625, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.125, 'completions/mean_terminated_length': 12.642857551574707, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.5, 'rewards/format_valid/std': 0.0, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': -0.04019365459680557, 'rewards/env_reward/std': 0.3211946189403534, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': -0.33529049158096313, 'reward_std': 0.1754429191350937, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.5859772562980652, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.23}
+
+
+ 14%|█▍        | 114/800 [38:23<3:32:21, 18.57s/it]
+
+
+ 14%|█▍        | 115/800 [38:38<3:23:03, 17.79s/it]
+
+
+                                                   
+
+
+{'loss': 0.0672, 'grad_norm': 1.1452234983444214, 'learning_rate': 4.7638888888888887e-05, 'num_tokens': 2285331.0, 'completions/mean_length': 14.65625, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.125, 'completions/mean_terminated_length': 12.178571701049805, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 24.0, 'rewards/format_valid/mean': 0.546875, 'rewards/format_valid/std': 0.5136739015579224, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.3019571602344513, 'rewards/env_reward/std': 0.6663445830345154, 'rewards/belief_accuracy/mean': -0.11784438788890839, 'rewards/belief_accuracy/std': 0.04648065194487572, 'reward': 0.1251838207244873, 'reward_std': 0.7151124477386475, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.6797190010547638, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.23}
+
+
+ 14%|█▍        | 115/800 [38:38<3:23:03, 17.79s/it]
+
+
+ 14%|█▍        | 116/800 [38:54<3:16:40, 17.25s/it]
+
+
+                                                   
+
+
+{'loss': 0.0723, 'grad_norm': 1.0745000839233398, 'learning_rate': 4.756944444444444e-05, 'num_tokens': 2305070.0, 'completions/mean_length': 16.84375, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.09375, 'completions/mean_terminated_length': 15.275861740112305, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 29.0, 'rewards/format_valid/mean': 0.5, 'rewards/format_valid/std': 0.0, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.024419229477643967, 'rewards/env_reward/std': 0.16250810027122498, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': -0.23837114870548248, 'reward_std': 0.051399923861026764, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.8075788617134094, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.23}
+
+
+ 14%|█▍        | 116/800 [38:54<3:16:40, 17.25s/it]
+
+{'loss': 0.0613, 'grad_norm': 1.2685792446136475, 'learning_rate': 4.75e-05, 'num_tokens': 2324718.0, 'completions/mean_length': 14.0, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.09375, 'completions/mean_terminated_length': 12.137930870056152, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 28.0, 'rewards/format_valid/mean': 0.1875, 'rewards/format_valid/std': 0.84002685546875, 'rewards/action_legal/mean': -0.125, 'rewards/action_legal/std': 0.33601075410842896, 'rewards/env_reward/mean': -0.113365538418293, 'rewards/env_reward/std': 1.2177670001983643, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': -0.466923326253891, 'reward_std': 1.5603351593017578, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.533688873052597, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.23}
+
+
+ 15%|█▍        | 117/800 [39:10<3:11:54, 16.86s/it]
+
+
+                                                   
+
+
+
+ 15%|█▍        | 117/800 [39:10<3:11:54, 16.86s/it]
+
+
+ 15%|█▍        | 118/800 [39:29<3:16:33, 17.29s/it]
+
+
+                                                   
+
+
+{'loss': 0.0623, 'grad_norm': 2.416207790374756, 'learning_rate': 4.743055555555556e-05, 'num_tokens': 2343965.0, 'completions/mean_length': 15.46875, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.1875, 'completions/mean_terminated_length': 11.653846740722656, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 24.0, 'rewards/format_valid/mean': 0.625, 'rewards/format_valid/std': 0.2199706733226776, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.45102953910827637, 'rewards/env_reward/std': 0.46820420026779175, 'rewards/belief_accuracy/mean': -0.07500000298023224, 'rewards/belief_accuracy/std': 0.04399413242936134, 'reward': 0.48279428482055664, 'reward_std': 0.2730714678764343, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.5577685832977295, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.24}
+
+
+ 15%|█▍        | 118/800 [39:29<3:16:33, 17.29s/it]
+
+{'loss': 0.0727, 'grad_norm': 1.1518934965133667, 'learning_rate': 4.736111111111111e-05, 'num_tokens': 2363664.0, 'completions/mean_length': 15.59375, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.09375, 'completions/mean_terminated_length': 13.896552085876465, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': 0.421875, 'rewards/format_valid/std': 0.4419417381286621, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.5008161067962646, 'rewards/env_reward/std': 0.7322836518287659, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': 0.470755398273468, 'reward_std': 0.7136918306350708, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.816842257976532, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.24}
+
+
+ 15%|█▍        | 119/800 [39:45<3:11:45, 16.90s/it]
+
+
+                                                   
+
+
+
+ 15%|█▍        | 119/800 [39:45<3:11:45, 16.90s/it]
+
+{'loss': 0.0591, 'grad_norm': 1.0581538677215576, 'learning_rate': 4.7291666666666666e-05, 'num_tokens': 2383324.0, 'completions/mean_length': 14.375, 'completions/min_length': 5.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.09375, 'completions/mean_terminated_length': 12.551724433898926, 'completions/min_terminated_length': 5.0, 'completions/max_terminated_length': 27.0, 'rewards/format_valid/mean': 0.109375, 'rewards/format_valid/std': 0.9222550392150879, 'rewards/action_legal/mean': -0.15625, 'rewards/action_legal/std': 0.3689020276069641, 'rewards/env_reward/mean': 0.005924493074417114, 'rewards/env_reward/std': 1.4021295309066772, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': -0.2934570014476776, 'reward_std': 1.555702805519104, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.4785886108875275, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.24}
+
+
+ 15%|█▌        | 120/800 [40:01<3:08:16, 16.61s/it]
+
+
+                                                   
+
+
+
+ 15%|█▌        | 120/800 [40:01<3:08:16, 16.61s/it]
+
+
+ 15%|█▌        | 121/800 [40:17<3:05:52, 16.43s/it]
+
+
+                                                   
+
+
+{'loss': 0.0813, 'grad_norm': 1.448180079460144, 'learning_rate': 4.722222222222222e-05, 'num_tokens': 2402896.0, 'completions/mean_length': 11.625, 'completions/min_length': 5.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.03125, 'completions/mean_terminated_length': 10.967741966247559, 'completions/min_terminated_length': 5.0, 'completions/max_terminated_length': 21.0, 'rewards/format_valid/mean': 0.34375, 'rewards/format_valid/std': 0.6148366928100586, 'rewards/action_legal/mean': -0.0625, 'rewards/action_legal/std': 0.24593468010425568, 'rewards/env_reward/mean': 0.10722772777080536, 'rewards/env_reward/std': 0.8865904211997986, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': -0.125095933675766, 'reward_std': 0.6908898949623108, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 2.032832622528076, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.24}
+
+
+ 15%|█▌        | 121/800 [40:17<3:05:52, 16.43s/it]
+
+{'loss': 0.072, 'grad_norm': 1.4070144891738892, 'learning_rate': 4.7152777777777776e-05, 'num_tokens': 2422532.0, 'completions/mean_length': 13.625, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.0625, 'completions/mean_terminated_length': 12.40000057220459, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 27.0, 'rewards/format_valid/mean': 0.34375, 'rewards/format_valid/std': 0.6148366928100586, 'rewards/action_legal/mean': -0.0625, 'rewards/action_legal/std': 0.24593468010425568, 'rewards/env_reward/mean': -0.07822468876838684, 'rewards/env_reward/std': 0.82816481590271, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': -0.4032745361328125, 'reward_std': 0.7969781160354614, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.799138993024826, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.24}
+
+
+ 15%|█▌        | 122/800 [40:33<3:04:01, 16.28s/it]
+
+
+                                                   
+
+
+
+ 15%|█▌        | 122/800 [40:33<3:04:01, 16.28s/it]
+
+
+ 15%|█▌        | 123/800 [40:49<3:02:43, 16.19s/it]
+
+
+                                                   
+
+
+{'loss': 0.0636, 'grad_norm': 1.1820019483566284, 'learning_rate': 4.708333333333334e-05, 'num_tokens': 2442153.0, 'completions/mean_length': 13.15625, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.125, 'completions/mean_terminated_length': 10.464285850524902, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 18.0, 'rewards/format_valid/mean': 0.421875, 'rewards/format_valid/std': 0.4419417381286621, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': -0.14212092757225037, 'rewards/env_reward/std': 0.7838878631591797, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': -0.4936501681804657, 'reward_std': 0.5655930042266846, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.589186191558838, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.25}
+
+
+ 15%|█▌        | 123/800 [40:49<3:02:43, 16.19s/it]
+
+{'loss': 0.0726, 'grad_norm': 0.5929283499717712, 'learning_rate': 4.701388888888889e-05, 'num_tokens': 2461694.0, 'completions/mean_length': 10.65625, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.03125, 'completions/mean_terminated_length': 9.967741966247559, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 26.0, 'rewards/format_valid/mean': 0.546875, 'rewards/format_valid/std': 0.5136739015579224, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.7039394378662109, 'rewards/env_reward/std': 0.7898678779602051, 'rewards/belief_accuracy/mean': -0.09185147285461426, 'rewards/belief_accuracy/std': 0.05036250501871109, 'reward': 0.8061359524726868, 'reward_std': 0.4916141927242279, 'frac_reward_zero_std': 0.5, 'completion_length': 32.0, 'kl': 1.8151837885379791, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.25}
+
+
+ 16%|█▌        | 124/800 [41:05<3:01:32, 16.11s/it]
+
+
+                                                   
+
+
+
+ 16%|█▌        | 124/800 [41:05<3:01:32, 16.11s/it]
+
+{'loss': 0.0615, 'grad_norm': 0.7959973812103271, 'learning_rate': 4.6944444444444446e-05, 'num_tokens': 2481327.0, 'completions/mean_length': 13.53125, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.09375, 'completions/mean_terminated_length': 11.620689392089844, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 24.0, 'rewards/format_valid/mean': 0.421875, 'rewards/format_valid/std': 0.4419417381286621, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.338913232088089, 'rewards/env_reward/std': 0.826060950756073, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': 0.22790104150772095, 'reward_std': 0.3862084448337555, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.5383435487747192, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.25}
+
+
+ 16%|█▌        | 125/800 [41:20<3:00:48, 16.07s/it]
+
+
+                                                   
+
+
+
+ 16%|█▌        | 125/800 [41:20<3:00:48, 16.07s/it]
+
+
+ 16%|█▌        | 126/800 [41:36<3:00:12, 16.04s/it]
+
+
+                                                   
+
+
+{'loss': 0.0653, 'grad_norm': 0.49789896607398987, 'learning_rate': 4.6875e-05, 'num_tokens': 2500878.0, 'completions/mean_length': 10.96875, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.03125, 'completions/mean_terminated_length': 10.290322303771973, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 19.0, 'rewards/format_valid/mean': 0.5, 'rewards/format_valid/std': 0.0, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.6813544631004333, 'rewards/env_reward/std': 0.4398447275161743, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': 0.7470316886901855, 'reward_std': 0.07455521076917648, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.6313612163066864, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.25}
+
+
+ 16%|█▌        | 126/800 [41:36<3:00:12, 16.04s/it]
+
+{'loss': 0.0717, 'grad_norm': 0.04532596096396446, 'learning_rate': 4.6805555555555556e-05, 'num_tokens': 2520535.0, 'completions/mean_length': 14.28125, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.125, 'completions/mean_terminated_length': 11.750000953674316, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 26.0, 'rewards/format_valid/mean': 0.5, 'rewards/format_valid/std': 0.0, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 1.1543023586273193, 'rewards/env_reward/std': 0.12598426640033722, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': 1.456453561782837, 'reward_std': 0.0, 'frac_reward_zero_std': 1.0, 'completion_length': 32.0, 'kl': 1.7934706807136536, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.25}
+
+
+ 16%|█▌        | 127/800 [41:52<2:59:49, 16.03s/it]
+
+
+                                                   
+
+
+
+ 16%|█▌        | 127/800 [41:52<2:59:49, 16.03s/it]
+
+
+ 16%|█▌        | 128/800 [42:08<2:59:12, 16.00s/it]
+
+
+                                                   
+
+
+{'loss': 0.0656, 'grad_norm': 0.03929617628455162, 'learning_rate': 4.673611111111112e-05, 'num_tokens': 2540090.0, 'completions/mean_length': 11.09375, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.03125, 'completions/mean_terminated_length': 10.419354438781738, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 22.0, 'rewards/format_valid/mean': 0.5, 'rewards/format_valid/std': 0.0, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.6777822971343994, 'rewards/env_reward/std': 0.28319674730300903, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': 0.7416733503341675, 'reward_std': 0.0, 'frac_reward_zero_std': 1.0, 'completion_length': 32.0, 'kl': 1.6403851211071014, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.26}
+
+
+ 16%|█▌        | 128/800 [42:08<2:59:12, 16.00s/it]
+
+
+ 16%|█▌        | 129/800 [42:24<2:58:50, 15.99s/it]
+
+
+                                                   
+
+
+{'loss': 0.0763, 'grad_norm': 0.09648065268993378, 'learning_rate': 4.666666666666667e-05, 'num_tokens': 2559686.0, 'completions/mean_length': 12.375, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.03125, 'completions/mean_terminated_length': 11.741934776306152, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.5, 'rewards/format_valid/std': 0.0, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.5401915907859802, 'rewards/env_reward/std': 0.457970529794693, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': 0.5352873206138611, 'reward_std': 0.0, 'frac_reward_zero_std': 1.0, 'completion_length': 32.0, 'kl': 1.9080970883369446, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.26}
+
+
+ 16%|█▌        | 129/800 [42:24<2:58:50, 15.99s/it]
+
+
+ 16%|█▋        | 130/800 [45:14<11:34:30, 62.19s/it]
+
+
+                                                    
+
+
+{'loss': 0.0888, 'grad_norm': 0.9380160570144653, 'learning_rate': 4.6597222222222226e-05, 'num_tokens': 2579187.0, 'completions/mean_length': 9.40625, 'completions/min_length': 6.0, 'completions/max_length': 23.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 9.40625, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 23.0, 'rewards/format_valid/mean': 0.546875, 'rewards/format_valid/std': 0.1480722874403, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.4952644407749176, 'rewards/env_reward/std': 0.8996369242668152, 'rewards/belief_accuracy/mean': -0.10427071154117584, 'rewards/belief_accuracy/std': 0.013490642420947552, 'reward': 0.4574282765388489, 'reward_std': 0.033855829387903214, 'frac_reward_zero_std': 0.75, 'completion_length': 23.0, 'kl': 2.2195359468460083, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.26}
+
+
+ 16%|█▋        | 130/800 [45:14<11:34:30, 62.19s/it]
+
+{'loss': 0.0845, 'grad_norm': 0.10833554714918137, 'learning_rate': 4.652777777777778e-05, 'num_tokens': 2598692.0, 'completions/mean_length': 9.53125, 'completions/min_length': 6.0, 'completions/max_length': 31.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 9.53125, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': 0.5, 'rewards/format_valid/std': 0.0, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.637738823890686, 'rewards/env_reward/std': 0.49828359484672546, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': 0.681608259677887, 'reward_std': 0.0, 'frac_reward_zero_std': 1.0, 'completion_length': 31.0, 'kl': 2.1125581860542297, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.26}
+
+
+ 16%|█▋        | 131/800 [47:26<15:26:17, 83.08s/it]
+
+
+                                                    
+
+
+
+ 16%|█▋        | 131/800 [47:26<15:26:17, 83.08s/it]
+
+
+ 16%|█▋        | 132/800 [49:33<17:50:00, 96.11s/it]
+
+
+                                                    
+
+
+{'loss': 0.1055, 'grad_norm': 0.5994073152542114, 'learning_rate': 4.6458333333333335e-05, 'num_tokens': 2618229.0, 'completions/mean_length': 10.53125, 'completions/min_length': 7.0, 'completions/max_length': 19.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 10.53125, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 19.0, 'rewards/format_valid/mean': 0.515625, 'rewards/format_valid/std': 0.0883883461356163, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.35123056173324585, 'rewards/env_reward/std': 0.5794245600700378, 'rewards/belief_accuracy/mean': -0.0991523265838623, 'rewards/belief_accuracy/std': 0.004795178771018982, 'reward': 0.2551701068878174, 'reward_std': 0.009402474388480186, 'frac_reward_zero_std': 0.75, 'completion_length': 19.0, 'kl': 2.638068199157715, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.26}
+
+
+ 16%|█▋        | 132/800 [49:33<17:50:00, 96.11s/it]
+
+{'loss': 0.0735, 'grad_norm': 0.7747211456298828, 'learning_rate': 4.638888888888889e-05, 'num_tokens': 2637338.0, 'completions/mean_length': 11.15625, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.03125, 'completions/mean_terminated_length': 10.483870506286621, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 22.0, 'rewards/format_valid/mean': 0.640625, 'rewards/format_valid/std': 0.22840170562267303, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.798025369644165, 'rewards/env_reward/std': 0.8904176950454712, 'rewards/belief_accuracy/mean': -0.07708333432674408, 'rewards/belief_accuracy/std': 0.04671080783009529, 'reward': 0.9978193044662476, 'reward_std': 0.07799539715051651, 'frac_reward_zero_std': 0.5, 'completion_length': 32.0, 'kl': 1.837352067232132, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.27}
+
+
+ 17%|█▋        | 133/800 [49:51<13:28:57, 72.77s/it]
+
+
+                                                    
+
+
+
+ 17%|█▋        | 133/800 [49:51<13:28:57, 72.77s/it]
+
+{'loss': 0.0967, 'grad_norm': 0.15459690988063812, 'learning_rate': 4.631944444444445e-05, 'num_tokens': 2656905.0, 'completions/mean_length': 11.46875, 'completions/min_length': 7.0, 'completions/max_length': 22.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 11.46875, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 22.0, 'rewards/format_valid/mean': 0.5, 'rewards/format_valid/std': 0.0, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': -0.013693984597921371, 'rewards/env_reward/std': 0.19833451509475708, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': -0.295540988445282, 'reward_std': 0.0, 'frac_reward_zero_std': 1.0, 'completion_length': 22.0, 'kl': 2.4168556332588196, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.27}
+
+
+ 17%|█▋        | 134/800 [52:36<18:35:02, 100.45s/it]
+
+
+                                                     
+
+
+
+ 17%|█▋        | 134/800 [52:36<18:35:02, 100.45s/it]
+
+{'loss': 0.0765, 'grad_norm': 0.7109292149543762, 'learning_rate': 4.6250000000000006e-05, 'num_tokens': 2676459.0, 'completions/mean_length': 11.0625, 'completions/min_length': 7.0, 'completions/max_length': 30.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 11.0625, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.578125, 'rewards/format_valid/std': 0.18445101380348206, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.5103276371955872, 'rewards/env_reward/std': 0.3578580617904663, 'rewards/belief_accuracy/mean': -0.09530973434448242, 'rewards/belief_accuracy/std': 0.03094319999217987, 'reward': 0.5084684491157532, 'reward_std': 0.2282516211271286, 'frac_reward_zero_std': 0.75, 'completion_length': 30.0, 'kl': 1.9128845036029816, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.27}
+
+
+ 17%|█▋        | 135/800 [54:42<19:59:14, 108.20s/it]
+
+
+                                                     
+
+
+
+ 17%|█▋        | 135/800 [54:42<19:59:14, 108.20s/it]
+
+
+ 17%|█▋        | 136/800 [54:58<14:51:17, 80.54s/it] 
+
+
+                                                    
+
+
+{'loss': 0.0803, 'grad_norm': 0.09752160310745239, 'learning_rate': 4.618055555555556e-05, 'num_tokens': 2696016.0, 'completions/mean_length': 11.15625, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.03125, 'completions/mean_terminated_length': 10.483870506286621, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 19.0, 'rewards/format_valid/mean': 0.5, 'rewards/format_valid/std': 0.0, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.20629703998565674, 'rewards/env_reward/std': 0.5170042514801025, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': 0.03444550931453705, 'reward_std': 0.0, 'frac_reward_zero_std': 1.0, 'completion_length': 32.0, 'kl': 2.0067217648029327, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.27}
+
+
+ 17%|█▋        | 136/800 [54:58<14:51:17, 80.54s/it]
+
+
+ 17%|█▋        | 137/800 [55:14<11:15:56, 61.17s/it]
+
+
+                                                    
+
+
+{'loss': 0.066, 'grad_norm': 0.12219260632991791, 'learning_rate': 4.6111111111111115e-05, 'num_tokens': 2715624.0, 'completions/mean_length': 12.75, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.09375, 'completions/mean_terminated_length': 10.758620262145996, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 18.0, 'rewards/format_valid/mean': 0.5, 'rewards/format_valid/std': 0.0, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.8520315885543823, 'rewards/env_reward/std': 0.6818674802780151, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': 1.0030474662780762, 'reward_std': 0.0, 'frac_reward_zero_std': 1.0, 'completion_length': 32.0, 'kl': 1.6505319476127625, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.27}
+
+
+ 17%|█▋        | 137/800 [55:14<11:15:56, 61.17s/it]
+
+{'loss': 0.08, 'grad_norm': 0.5224131345748901, 'learning_rate': 4.604166666666666e-05, 'num_tokens': 2735276.0, 'completions/mean_length': 14.125, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.03125, 'completions/mean_terminated_length': 13.548386573791504, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 28.0, 'rewards/format_valid/mean': 0.515625, 'rewards/format_valid/std': 0.0883883461356163, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.9824394583702087, 'rewards/env_reward/std': 0.6768037676811218, 'rewards/belief_accuracy/mean': -0.1002083420753479, 'rewards/belief_accuracy/std': 0.0011785115348175168, 'reward': 1.1988153457641602, 'reward_std': 0.0004419398319441825, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 2.000370144844055, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.28}
+
+
+ 17%|█▋        | 138/800 [55:30<8:45:32, 47.63s/it] 
+
+
+                                                   
+
+
+
+ 17%|█▋        | 138/800 [55:30<8:45:32, 47.63s/it]
+
+{'loss': 0.0671, 'grad_norm': 1.2123395204544067, 'learning_rate': 4.5972222222222225e-05, 'num_tokens': 2754906.0, 'completions/mean_length': 13.4375, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.0625, 'completions/mean_terminated_length': 12.200000762939453, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 23.0, 'rewards/format_valid/mean': 0.59375, 'rewards/format_valid/std': 0.19827888906002045, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.5191921591758728, 'rewards/env_reward/std': 0.3795930743217468, 'rewards/belief_accuracy/mean': -0.09519253671169281, 'rewards/belief_accuracy/std': 0.06183299422264099, 'reward': 0.5228981971740723, 'reward_std': 0.17345157265663147, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.6786936521530151, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.28}
+
+
+ 17%|█▋        | 139/800 [55:46<7:00:07, 38.14s/it]
+
+
+                                                   
+
+
+
+ 17%|█▋        | 139/800 [55:46<7:00:07, 38.14s/it]
+
+
+ 18%|█▊        | 140/800 [56:02<5:46:20, 31.49s/it]
+
+
+                                                   
+
+
+{'loss': 0.0641, 'grad_norm': 0.6268853545188904, 'learning_rate': 4.590277777777778e-05, 'num_tokens': 2774607.0, 'completions/mean_length': 15.65625, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.125, 'completions/mean_terminated_length': 13.321429252624512, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 23.0, 'rewards/format_valid/mean': 0.5625, 'rewards/format_valid/std': 0.16800537705421448, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 1.592028260231018, 'rewards/env_reward/std': 0.35845524072647095, 'rewards/belief_accuracy/mean': -0.09782090783119202, 'rewards/belief_accuracy/std': 0.0372542105615139, 'reward': 2.1227047443389893, 'reward_std': 0.07207237184047699, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.601601243019104, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.28}
+
+
+ 18%|█▊        | 140/800 [56:02<5:46:20, 31.49s/it]
+
+{'loss': 0.07, 'grad_norm': 0.1342216432094574, 'learning_rate': 4.5833333333333334e-05, 'num_tokens': 2794253.0, 'completions/mean_length': 13.9375, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.03125, 'completions/mean_terminated_length': 13.354838371276855, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 28.0, 'rewards/format_valid/mean': 0.5, 'rewards/format_valid/std': 0.0, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.4671225845813751, 'rewards/env_reward/std': 0.5395122766494751, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': 0.4256838858127594, 'reward_std': 0.0, 'frac_reward_zero_std': 1.0, 'completion_length': 32.0, 'kl': 1.7510676681995392, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.28}
+
+
+ 18%|█▊        | 141/800 [56:18<4:54:44, 26.84s/it]
+
+
+                                                   
+
+
+
+ 18%|█▊        | 141/800 [56:18<4:54:44, 26.84s/it]
+
+{'loss': 0.071, 'grad_norm': 1.0334209203720093, 'learning_rate': 4.576388888888889e-05, 'num_tokens': 2813915.0, 'completions/mean_length': 14.4375, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.125, 'completions/mean_terminated_length': 11.928571701049805, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.625, 'rewards/format_valid/std': 0.2199706733226776, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.9741031527519226, 'rewards/env_reward/std': 0.511637270450592, 'rewards/belief_accuracy/mean': -0.12225492298603058, 'rewards/belief_accuracy/std': 0.07321155071258545, 'reward': 1.1256399154663086, 'reward_std': 0.13475561141967773, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.7752875089645386, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.28}
+
+
+ 18%|█▊        | 142/800 [56:34<4:18:37, 23.58s/it]
+
+
+                                                   
+
+
+
+ 18%|█▊        | 142/800 [56:34<4:18:37, 23.58s/it]
+
+
+ 18%|█▊        | 143/800 [56:50<3:53:18, 21.31s/it]
+
+
+                                                   
+
+
+{'loss': 0.0574, 'grad_norm': 0.6873443126678467, 'learning_rate': 4.569444444444444e-05, 'num_tokens': 2833683.0, 'completions/mean_length': 17.75, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.125, 'completions/mean_terminated_length': 15.714286804199219, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': 0.625, 'rewards/format_valid/std': 0.2199706733226776, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.79294353723526, 'rewards/env_reward/std': 0.5636472702026367, 'rewards/belief_accuracy/mean': -0.1285519152879715, 'rewards/belief_accuracy/std': 0.052625637501478195, 'reward': 0.8350095748901367, 'reward_std': 0.12390097230672836, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.434746116399765, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.29}
+
+
+ 18%|█▊        | 143/800 [56:50<3:53:18, 21.31s/it]
+
+
+ 18%|█▊        | 144/800 [57:06<3:35:38, 19.72s/it]
+
+
+                                                   
+
+
+{'loss': 0.0762, 'grad_norm': 0.16623857617378235, 'learning_rate': 4.5625e-05, 'num_tokens': 2853375.0, 'completions/mean_length': 15.375, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.09375, 'completions/mean_terminated_length': 13.655172348022461, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 26.0, 'rewards/format_valid/mean': 0.5, 'rewards/format_valid/std': 0.0, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.37501204013824463, 'rewards/env_reward/std': 0.3454889953136444, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': 0.2875180244445801, 'reward_std': 0.0, 'frac_reward_zero_std': 1.0, 'completion_length': 32.0, 'kl': 1.9044136106967926, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.29}
+
+
+ 18%|█▊        | 144/800 [57:06<3:35:38, 19.72s/it]
+
+
+ 18%|█▊        | 145/800 [57:22<3:23:05, 18.60s/it]
+
+
+                                                   
+
+
+{'loss': 0.0614, 'grad_norm': 1.0505189895629883, 'learning_rate': 4.555555555555556e-05, 'num_tokens': 2873062.0, 'completions/mean_length': 15.21875, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.125, 'completions/mean_terminated_length': 12.821429252624512, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.515625, 'rewards/format_valid/std': 0.0883883461356163, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.740533709526062, 'rewards/env_reward/std': 0.49054640531539917, 'rewards/belief_accuracy/mean': -0.09749999642372131, 'rewards/belief_accuracy/std': 0.014142136089503765, 'reward': 0.8440817594528198, 'reward_std': 0.023422913625836372, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.5343632102012634, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.29}
+
+
+ 18%|█▊        | 145/800 [57:22<3:23:05, 18.60s/it]
+
+
+ 18%|█▊        | 146/800 [57:38<3:14:17, 17.83s/it]
+
+
+                                                   
+
+
+{'loss': 0.051, 'grad_norm': 1.5774600505828857, 'learning_rate': 4.5486111111111114e-05, 'num_tokens': 2892819.0, 'completions/mean_length': 17.40625, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.15625, 'completions/mean_terminated_length': 14.703703880310059, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 25.0, 'rewards/format_valid/mean': 0.65625, 'rewards/format_valid/std': 0.23546454310417175, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.4501720666885376, 'rewards/env_reward/std': 0.5856297016143799, 'rewards/belief_accuracy/mean': -0.10096347332000732, 'rewards/belief_accuracy/std': 0.04606204852461815, 'reward': 0.4051801562309265, 'reward_std': 0.08541367948055267, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.2751143276691437, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.29}
+
+
+ 18%|█▊        | 146/800 [57:38<3:14:17, 17.83s/it]
+
+{'loss': 0.0425, 'grad_norm': 2.2620859146118164, 'learning_rate': 4.541666666666667e-05, 'num_tokens': 2912581.0, 'completions/mean_length': 17.5625, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.09375, 'completions/mean_terminated_length': 16.068965911865234, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 29.0, 'rewards/format_valid/mean': 0.65625, 'rewards/format_valid/std': 0.23546454310417175, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.57562255859375, 'rewards/env_reward/std': 0.40924084186553955, 'rewards/belief_accuracy/mean': -0.11185985803604126, 'rewards/belief_accuracy/std': 0.03135733678936958, 'reward': 0.5606666803359985, 'reward_std': 0.18964077532291412, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.0627762079238892, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.29}
+
+
+ 18%|█▊        | 147/800 [57:54<3:08:02, 17.28s/it]
+
+
+                                                   
+
+
+
+ 18%|█▊        | 147/800 [57:54<3:08:02, 17.28s/it]
+
+
+ 18%|█▊        | 148/800 [58:10<3:03:37, 16.90s/it]
+
+
+                                                   
+
+
+{'loss': 0.0525, 'grad_norm': 1.5949054956436157, 'learning_rate': 4.534722222222222e-05, 'num_tokens': 2932298.0, 'completions/mean_length': 16.15625, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.25, 'completions/mean_terminated_length': 10.875, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 22.0, 'rewards/format_valid/mean': 0.515625, 'rewards/format_valid/std': 0.0883883461356163, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.2627526521682739, 'rewards/env_reward/std': 0.702940821647644, 'rewards/belief_accuracy/mean': -0.1015625, 'rewards/belief_accuracy/std': 0.008838835172355175, 'reward': 0.11522269248962402, 'reward_std': 0.011048545129597187, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.3134159445762634, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.3}
+
+
+ 18%|█▊        | 148/800 [58:10<3:03:37, 16.90s/it]
+
+
+ 19%|█▊        | 149/800 [58:26<3:00:26, 16.63s/it]
+
+
+                                                   
+
+
+{'loss': 0.0606, 'grad_norm': 2.8204798698425293, 'learning_rate': 4.527777777777778e-05, 'num_tokens': 2952045.0, 'completions/mean_length': 17.09375, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.21875, 'completions/mean_terminated_length': 12.920000076293945, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 29.0, 'rewards/format_valid/mean': 0.515625, 'rewards/format_valid/std': 0.0883883461356163, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.7818121314048767, 'rewards/env_reward/std': 0.4523349106311798, 'rewards/belief_accuracy/mean': -0.10208334028720856, 'rewards/belief_accuracy/std': 0.01178511418402195, 'reward': 0.8922492861747742, 'reward_std': 0.041471511125564575, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.5157608836889267, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.3}
+
+
+ 19%|█▊        | 149/800 [58:26<3:00:26, 16.63s/it]
+
+
+ 19%|█▉        | 150/800 [58:42<2:58:05, 16.44s/it]
+
+
+                                                   
+
+
+{'loss': 0.1458, 'grad_norm': 12.819677352905273, 'learning_rate': 4.520833333333334e-05, 'num_tokens': 2971805.0, 'completions/mean_length': 17.5, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.1875, 'completions/mean_terminated_length': 14.153846740722656, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 26.0, 'rewards/format_valid/mean': 0.5, 'rewards/format_valid/std': 0.0, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.6779667735099792, 'rewards/env_reward/std': 0.430367648601532, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': 0.7419501543045044, 'reward_std': 0.16406486928462982, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 3.6450346410274506, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.3}
+
+
+ 19%|█▉        | 150/800 [58:42<2:58:05, 16.44s/it]
+
+
+ 19%|█▉        | 151/800 [58:58<2:56:19, 16.30s/it]
+
+
+                                                   
+
+
+{'loss': 0.0505, 'grad_norm': 6.00082540512085, 'learning_rate': 4.5138888888888894e-05, 'num_tokens': 2991645.0, 'completions/mean_length': 20.0, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.15625, 'completions/mean_terminated_length': 17.77777862548828, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 29.0, 'rewards/format_valid/mean': 0.765625, 'rewards/format_valid/std': 0.253503680229187, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.5839787125587463, 'rewards/env_reward/std': 0.449224054813385, 'rewards/belief_accuracy/mean': -0.12783914804458618, 'rewards/belief_accuracy/std': 0.037070371210575104, 'reward': 0.5307319164276123, 'reward_std': 0.13287778198719025, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.2636119723320007, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.3}
+
+
+ 19%|█▉        | 151/800 [58:58<2:56:19, 16.30s/it]
+
+{'loss': 0.0582, 'grad_norm': 2.6323814392089844, 'learning_rate': 4.506944444444445e-05, 'num_tokens': 3010947.0, 'completions/mean_length': 17.1875, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.15625, 'completions/mean_terminated_length': 14.44444465637207, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': 0.546875, 'rewards/format_valid/std': 0.5136739015579224, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.3130866289138794, 'rewards/env_reward/std': 0.7245592474937439, 'rewards/belief_accuracy/mean': -0.07500000298023224, 'rewards/belief_accuracy/std': 0.04399413242936134, 'reward': 0.27041125297546387, 'reward_std': 0.8563976287841797, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.4539388418197632, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.3}
+
+
+ 19%|█▉        | 152/800 [59:17<3:02:37, 16.91s/it]
+
+
+                                                   
+
+
+
+ 19%|█▉        | 152/800 [59:17<3:02:37, 16.91s/it]
+
+
+ 19%|█▉        | 153/800 [59:33<2:59:16, 16.62s/it]
+
+
+                                                   
+
+
+{'loss': 0.0535, 'grad_norm': 2.7524871826171875, 'learning_rate': 4.5e-05, 'num_tokens': 3030665.0, 'completions/mean_length': 16.1875, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.15625, 'completions/mean_terminated_length': 13.259259223937988, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 25.0, 'rewards/format_valid/mean': 0.609375, 'rewards/format_valid/std': 0.7266180515289307, 'rewards/action_legal/mean': -0.0625, 'rewards/action_legal/std': 0.24593468010425568, 'rewards/env_reward/mean': 0.8374850749969482, 'rewards/env_reward/std': 1.0421284437179565, 'rewards/belief_accuracy/mean': -0.12535476684570312, 'rewards/belief_accuracy/std': 0.07478667050600052, 'reward': 0.907507061958313, 'reward_std': 0.8716118335723877, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.3364153802394867, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.31}
+
+
+ 19%|█▉        | 153/800 [59:33<2:59:16, 16.62s/it]
+
+{'loss': 0.0859, 'grad_norm': 62.430763244628906, 'learning_rate': 4.493055555555556e-05, 'num_tokens': 3050551.0, 'completions/mean_length': 21.4375, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.34375, 'completions/mean_terminated_length': 15.904762268066406, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 29.0, 'rewards/format_valid/mean': 0.40625, 'rewards/format_valid/std': 0.817524790763855, 'rewards/action_legal/mean': -0.09375, 'rewards/action_legal/std': 0.2961445748806, 'rewards/env_reward/mean': 0.7318300008773804, 'rewards/env_reward/std': 1.4541698694229126, 'rewards/belief_accuracy/mean': -0.10801082104444504, 'rewards/belief_accuracy/std': 0.06327005475759506, 'reward': 0.7893375158309937, 'reward_std': 1.2687653303146362, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 2.1476181596517563, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.31}
+
+
+ 19%|█▉        | 154/800 [59:49<2:56:55, 16.43s/it]
+
+
+                                                   
+
+
+
+ 19%|█▉        | 154/800 [59:49<2:56:55, 16.43s/it]
+
+
+ 19%|█▉        | 155/800 [1:00:05<2:55:35, 16.33s/it]
+
+
+                                                     
+
+
+{'loss': 0.0395, 'grad_norm': 12.50893497467041, 'learning_rate': 4.486111111111111e-05, 'num_tokens': 3070373.0, 'completions/mean_length': 19.4375, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.3125, 'completions/mean_terminated_length': 13.727272987365723, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 23.0, 'rewards/format_valid/mean': 0.4375, 'rewards/format_valid/std': 0.45348116755485535, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.3174271881580353, 'rewards/env_reward/std': 0.6438456177711487, 'rewards/belief_accuracy/mean': -0.09861111640930176, 'rewards/belief_accuracy/std': 0.007856741547584534, 'reward': 0.2006199210882187, 'reward_std': 0.48218274116516113, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 0.9871228933334351, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.31}
+
+
+ 19%|█▉        | 155/800 [1:00:05<2:55:35, 16.33s/it]
+
+
+ 20%|█▉        | 156/800 [1:00:23<3:01:44, 16.93s/it]
+
+
+                                                     
+
+
+{'loss': 0.0415, 'grad_norm': 2.5396976470947266, 'learning_rate': 4.4791666666666673e-05, 'num_tokens': 3089831.0, 'completions/mean_length': 22.0625, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.375, 'completions/mean_terminated_length': 16.100000381469727, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.40625, 'rewards/format_valid/std': 0.817524790763855, 'rewards/action_legal/mean': -0.09375, 'rewards/action_legal/std': 0.2961445748806, 'rewards/env_reward/mean': -0.08344705402851105, 'rewards/env_reward/std': 0.9975131750106812, 'rewards/belief_accuracy/mean': -0.08114583790302277, 'rewards/belief_accuracy/std': 0.04542597755789757, 'reward': -0.3529830873012543, 'reward_std': 1.11542546749115, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.0387186855077744, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.31}
+
+
+ 20%|█▉        | 156/800 [1:00:23<3:01:44, 16.93s/it]
+
+
+ 20%|█▉        | 157/800 [1:00:39<2:58:24, 16.65s/it]
+
+
+                                                     
+
+
+{'loss': 0.0267, 'grad_norm': 1.5496879816055298, 'learning_rate': 4.472222222222223e-05, 'num_tokens': 3109936.0, 'completions/mean_length': 28.28125, 'completions/min_length': 9.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.65625, 'completions/mean_terminated_length': 21.18181800842285, 'completions/min_terminated_length': 9.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.34375, 'rewards/format_valid/std': 0.6148366928100586, 'rewards/action_legal/mean': -0.0625, 'rewards/action_legal/std': 0.24593468010425568, 'rewards/env_reward/mean': 0.30998486280441284, 'rewards/env_reward/std': 0.882105827331543, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': 0.17903976142406464, 'reward_std': 0.9768354892730713, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 0.6682075709104538, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.31}
+
+
+ 20%|█▉        | 157/800 [1:00:39<2:58:24, 16.65s/it]
+
+{'loss': 0.042, 'grad_norm': 3.7796595096588135, 'learning_rate': 4.465277777777778e-05, 'num_tokens': 3129905.0, 'completions/mean_length': 24.03125, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.46875, 'completions/mean_terminated_length': 17.0, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 29.0, 'rewards/format_valid/mean': 0.421875, 'rewards/format_valid/std': 0.4419417381286621, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.5461791157722473, 'rewards/env_reward/std': 0.9121921062469482, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': 0.5388000011444092, 'reward_std': 0.6551840901374817, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.0511186122894287, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.32}
+
+
+ 20%|█▉        | 158/800 [1:00:55<2:56:06, 16.46s/it]
+
+
+                                                     
+
+
+
+ 20%|█▉        | 158/800 [1:00:55<2:56:06, 16.46s/it]
+
+
+ 20%|█▉        | 159/800 [1:01:11<2:54:12, 16.31s/it]
+
+
+                                                     
+
+
+{'loss': 0.0285, 'grad_norm': 9.656961441040039, 'learning_rate': 4.458333333333334e-05, 'num_tokens': 3149869.0, 'completions/mean_length': 23.875, 'completions/min_length': 10.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.5, 'completions/mean_terminated_length': 15.75, 'completions/min_terminated_length': 10.0, 'completions/max_terminated_length': 26.0, 'rewards/format_valid/mean': 0.4375, 'rewards/format_valid/std': 0.45348116755485535, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.5372031331062317, 'rewards/env_reward/std': 0.6755579710006714, 'rewards/belief_accuracy/mean': -0.09878861904144287, 'rewards/belief_accuracy/std': 0.006852575112134218, 'reward': 0.5297513008117676, 'reward_std': 0.6674039959907532, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 0.711479052901268, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.32}
+
+
+ 20%|█▉        | 159/800 [1:01:11<2:54:12, 16.31s/it]
+
+
+ 20%|██        | 160/800 [1:01:27<2:52:51, 16.21s/it]
+
+
+                                                     
+
+
+{'loss': 0.0328, 'grad_norm': 2.401477575302124, 'learning_rate': 4.4513888888888885e-05, 'num_tokens': 3169852.0, 'completions/mean_length': 24.46875, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.46875, 'completions/mean_terminated_length': 17.823530197143555, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': 0.484375, 'rewards/format_valid/std': 0.6896045207977295, 'rewards/action_legal/mean': -0.0625, 'rewards/action_legal/std': 0.24593468010425568, 'rewards/env_reward/mean': 0.8105643391609192, 'rewards/env_reward/std': 1.0092358589172363, 'rewards/belief_accuracy/mean': -0.10525794327259064, 'rewards/belief_accuracy/std': 0.05371677875518799, 'reward': 0.9211664199829102, 'reward_std': 1.223477840423584, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.8190625458955765, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.32}
+
+
+ 20%|██        | 160/800 [1:01:27<2:52:51, 16.21s/it]
+
+
+ 20%|██        | 161/800 [1:01:43<2:51:59, 16.15s/it]
+
+
+                                                     
+
+
+{'loss': 0.0568, 'grad_norm': 2.1067185401916504, 'learning_rate': 4.4444444444444447e-05, 'num_tokens': 3189675.0, 'completions/mean_length': 19.46875, 'completions/min_length': 4.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.25, 'completions/mean_terminated_length': 15.291666984558105, 'completions/min_terminated_length': 4.0, 'completions/max_terminated_length': 28.0, 'rewards/format_valid/mean': 0.375, 'rewards/format_valid/std': 0.6350005865097046, 'rewards/action_legal/mean': -0.0625, 'rewards/action_legal/std': 0.24593468010425568, 'rewards/env_reward/mean': 0.4844372272491455, 'rewards/env_reward/std': 0.9781147837638855, 'rewards/belief_accuracy/mean': -0.09872794151306152, 'rewards/belief_accuracy/std': 0.01452599372714758, 'reward': 0.44609692692756653, 'reward_std': 1.1091150045394897, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.4203459024429321, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.32}
+
+
+ 20%|██        | 161/800 [1:01:43<2:51:59, 16.15s/it]
+
+
+ 20%|██        | 162/800 [1:02:01<2:58:39, 16.80s/it]
+
+{'loss': 0.0397, 'grad_norm': 31.847930908203125, 'learning_rate': 4.4375e-05, 'num_tokens': 3209065.0, 'completions/mean_length': 19.9375, 'completions/min_length': 9.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.25, 'completions/mean_terminated_length': 15.916666984558105, 'completions/min_terminated_length': 9.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.328125, 'rewards/format_valid/std': 0.9211613535881042, 'rewards/action_legal/mean': -0.125, 'rewards/action_legal/std': 0.33601075410842896, 'rewards/env_reward/mean': -0.08248895406723022, 'rewards/env_reward/std': 1.226813793182373, 'rewards/belief_accuracy/mean': -0.08113425970077515, 'rewards/belief_accuracy/std': 0.045413173735141754, 'reward': -0.35697996616363525, 'reward_std': 1.4437724351882935, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 0.9920024126768112, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.32}
+
+
+                                                     
+
+
+
+ 20%|██        | 162/800 [1:02:01<2:58:39, 16.80s/it]
+
+{'loss': 0.0413, 'grad_norm': 8.838136672973633, 'learning_rate': 4.4305555555555556e-05, 'num_tokens': 3228981.0, 'completions/mean_length': 22.375, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.40625, 'completions/mean_terminated_length': 15.789473533630371, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': 0.375, 'rewards/format_valid/std': 0.6350005865097046, 'rewards/action_legal/mean': -0.0625, 'rewards/action_legal/std': 0.24593468010425568, 'rewards/env_reward/mean': 0.19750605523586273, 'rewards/env_reward/std': 0.8624459505081177, 'rewards/belief_accuracy/mean': -0.09369213134050369, 'rewards/belief_accuracy/std': 0.02522020787000656, 'reward': 0.03080766648054123, 'reward_std': 0.9838901162147522, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.0335555523633957, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.33}
+
+
+ 20%|██        | 163/800 [1:02:17<2:55:43, 16.55s/it]
+
+
+                                                     
+
+
+
+ 20%|██        | 163/800 [1:02:17<2:55:43, 16.55s/it]
+
+
+ 20%|██        | 164/800 [1:02:33<2:53:38, 16.38s/it]
+
+
+                                                     
+
+
+{'loss': 0.0492, 'grad_norm': 3.2258176803588867, 'learning_rate': 4.423611111111111e-05, 'num_tokens': 3248852.0, 'completions/mean_length': 20.96875, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.28125, 'completions/mean_terminated_length': 16.65217399597168, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.421875, 'rewards/format_valid/std': 0.4419417381286621, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.7876648306846619, 'rewards/env_reward/std': 0.7834634184837341, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': 0.9010283946990967, 'reward_std': 0.5502346158027649, 'frac_reward_zero_std': 0.5, 'completion_length': 32.0, 'kl': 1.2308619320392609, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.33}
+
+
+ 20%|██        | 164/800 [1:02:33<2:53:38, 16.38s/it]
+
+
+ 21%|██        | 165/800 [1:02:49<2:52:18, 16.28s/it]
+
+
+                                                     
+
+
+{'loss': 0.0383, 'grad_norm': 35.84771728515625, 'learning_rate': 4.4166666666666665e-05, 'num_tokens': 3268747.0, 'completions/mean_length': 21.71875, 'completions/min_length': 10.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.3125, 'completions/mean_terminated_length': 17.045455932617188, 'completions/min_terminated_length': 10.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.453125, 'rewards/format_valid/std': 0.4641915261745453, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.4818274676799774, 'rewards/env_reward/std': 0.8541983366012573, 'rewards/belief_accuracy/mean': -0.09322916716337204, 'rewards/belief_accuracy/std': 0.026726871728897095, 'reward': 0.4641473889350891, 'reward_std': 0.7782288193702698, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.9583852589130402, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.33}
+
+
+ 21%|██        | 165/800 [1:02:49<2:52:18, 16.28s/it]
+
+{'loss': 0.0451, 'grad_norm': 1.8733093738555908, 'learning_rate': 4.4097222222222226e-05, 'num_tokens': 3288156.0, 'completions/mean_length': 20.53125, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.25, 'completions/mean_terminated_length': 16.70833396911621, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': 0.640625, 'rewards/format_valid/std': 0.22840170562267303, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.12140398472547531, 'rewards/env_reward/std': 0.3865126669406891, 'rewards/belief_accuracy/mean': -0.07503403723239899, 'rewards/belief_accuracy/std': 0.04401451721787453, 'reward': -0.01096487045288086, 'reward_std': 0.08910165727138519, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.1273473799228668, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.33}
+
+
+ 21%|██        | 166/800 [1:03:08<2:58:31, 16.89s/it]
+
+
+                                                     
+
+
+
+ 21%|██        | 166/800 [1:03:08<2:58:31, 16.89s/it]
+
+{'loss': 0.0487, 'grad_norm': 1.4066689014434814, 'learning_rate': 4.402777777777778e-05, 'num_tokens': 3307987.0, 'completions/mean_length': 19.71875, 'completions/min_length': 10.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.21875, 'completions/mean_terminated_length': 16.279998779296875, 'completions/min_terminated_length': 10.0, 'completions/max_terminated_length': 23.0, 'rewards/format_valid/mean': 0.375, 'rewards/format_valid/std': 0.6350005865097046, 'rewards/action_legal/mean': -0.0625, 'rewards/action_legal/std': 0.24593468010425568, 'rewards/env_reward/mean': 0.4588669538497925, 'rewards/env_reward/std': 0.9698508977890015, 'rewards/belief_accuracy/mean': -0.09875000268220901, 'rewards/belief_accuracy/std': 0.008370882831513882, 'reward': 0.40767550468444824, 'reward_std': 1.0276349782943726, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.2167561650276184, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.33}
+
+
+ 21%|██        | 167/800 [1:03:24<2:55:23, 16.62s/it]
+
+
+                                                     
+
+
+
+ 21%|██        | 167/800 [1:03:24<2:55:23, 16.62s/it]
+
+{'loss': 0.0464, 'grad_norm': 1.0989431142807007, 'learning_rate': 4.3958333333333336e-05, 'num_tokens': 3327905.0, 'completions/mean_length': 22.4375, 'completions/min_length': 9.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.40625, 'completions/mean_terminated_length': 15.894737243652344, 'completions/min_terminated_length': 9.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.390625, 'rewards/format_valid/std': 0.6442596912384033, 'rewards/action_legal/mean': -0.0625, 'rewards/action_legal/std': 0.24593468010425568, 'rewards/env_reward/mean': 0.011147502809762955, 'rewards/env_reward/std': 1.0481460094451904, 'rewards/belief_accuracy/mean': -0.10723080486059189, 'rewards/belief_accuracy/std': 0.02330031991004944, 'reward': -0.28856492042541504, 'reward_std': 0.9179288148880005, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.1604894697666168, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.34}
+
+
+ 21%|██        | 168/800 [1:03:40<2:53:05, 16.43s/it]
+
+
+                                                     
+
+
+
+ 21%|██        | 168/800 [1:03:40<2:53:05, 16.43s/it]
+
+
+ 21%|██        | 169/800 [1:03:56<2:51:24, 16.30s/it]
+
+
+                                                     
+
+
+{'loss': 0.0512, 'grad_norm': 12.306889533996582, 'learning_rate': 4.388888888888889e-05, 'num_tokens': 3347827.0, 'completions/mean_length': 22.5625, 'completions/min_length': 9.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.4375, 'completions/mean_terminated_length': 15.222222328186035, 'completions/min_terminated_length': 9.0, 'completions/max_terminated_length': 26.0, 'rewards/format_valid/mean': 0.5, 'rewards/format_valid/std': 0.6956083178520203, 'rewards/action_legal/mean': -0.0625, 'rewards/action_legal/std': 0.24593468010425568, 'rewards/env_reward/mean': 0.5058934092521667, 'rewards/env_reward/std': 1.2340750694274902, 'rewards/belief_accuracy/mean': -0.06489582359790802, 'rewards/belief_accuracy/std': 0.06666442006826401, 'reward': 0.5860276818275452, 'reward_std': 0.721314549446106, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.2812400460243225, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.34}
+
+
+ 21%|██        | 169/800 [1:03:56<2:51:24, 16.30s/it]
+
+
+ 21%|██▏       | 170/800 [1:04:12<2:50:11, 16.21s/it]
+
+
+                                                     
+
+
+{'loss': 0.0449, 'grad_norm': 3.3559019565582275, 'learning_rate': 4.3819444444444445e-05, 'num_tokens': 3367742.0, 'completions/mean_length': 22.34375, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.375, 'completions/mean_terminated_length': 16.55000114440918, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 28.0, 'rewards/format_valid/mean': 0.40625, 'rewards/format_valid/std': 0.6530017256736755, 'rewards/action_legal/mean': -0.0625, 'rewards/action_legal/std': 0.24593468010425568, 'rewards/env_reward/mean': 0.12099152058362961, 'rewards/env_reward/std': 0.9435498118400574, 'rewards/belief_accuracy/mean': -0.09155841171741486, 'rewards/belief_accuracy/std': 0.04061589017510414, 'reward': -0.07600045204162598, 'reward_std': 0.9773452877998352, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.1225250661373138, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.34}
+
+
+ 21%|██▏       | 170/800 [1:04:12<2:50:11, 16.21s/it]
+
+
+ 21%|██▏       | 171/800 [1:04:28<2:49:19, 16.15s/it]
+
+
+                                                     
+
+
+{'loss': 0.0541, 'grad_norm': 1.9631366729736328, 'learning_rate': 4.375e-05, 'num_tokens': 3387600.0, 'completions/mean_length': 20.5625, 'completions/min_length': 9.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.28125, 'completions/mean_terminated_length': 16.086956024169922, 'completions/min_terminated_length': 9.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.578125, 'rewards/format_valid/std': 0.5253167152404785, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.4206196665763855, 'rewards/env_reward/std': 0.7749525308609009, 'rewards/belief_accuracy/mean': -0.10358966141939163, 'rewards/belief_accuracy/std': 0.05899786204099655, 'reward': 0.347504198551178, 'reward_std': 0.7113288640975952, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.352888435125351, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.34}
+
+
+ 21%|██▏       | 171/800 [1:04:28<2:49:19, 16.15s/it]
+
+
+ 22%|██▏       | 172/800 [1:04:44<2:48:29, 16.10s/it]
+
+
+                                                     
+
+
+{'loss': 0.0516, 'grad_norm': 4.23384952545166, 'learning_rate': 4.368055555555556e-05, 'num_tokens': 3407482.0, 'completions/mean_length': 21.3125, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.28125, 'completions/mean_terminated_length': 17.130435943603516, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': 0.625, 'rewards/format_valid/std': 0.2199706733226776, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.9689656496047974, 'rewards/env_reward/std': 0.6357868313789368, 'rewards/belief_accuracy/mean': -0.0872916728258133, 'rewards/belief_accuracy/std': 0.03746862709522247, 'reward': 1.2228235006332397, 'reward_std': 0.20254021883010864, 'frac_reward_zero_std': 0.5, 'completion_length': 32.0, 'kl': 1.2912044823169708, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.34}
+
+{'loss': 0.0558, 'grad_norm': 1.1539499759674072, 'learning_rate': 4.3611111111111116e-05, 'num_tokens': 3427250.0, 'completions/mean_length': 17.75, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.15625, 'completions/mean_terminated_length': 15.111111640930176, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.4375, 'rewards/format_valid/std': 0.45348116755485535, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.13206146657466888, 'rewards/env_reward/std': 0.8628634810447693, 'rewards/belief_accuracy/mean': -0.10208334028720856, 'rewards/belief_accuracy/std': 0.01178511418402195, 'reward': -0.08784535527229309, 'reward_std': 0.33973684906959534, 'frac_reward_zero_std': 0.5, 'completion_length': 32.0, 'kl': 1.3939580619335175, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.35}
+
+
+ 22%|██▏       | 172/800 [1:04:44<2:48:29, 16.10s/it]
+
+
+ 22%|██▏       | 173/800 [1:05:00<2:47:52, 16.07s/it]
+
+
+                                                     
+
+
+
+ 22%|██▏       | 173/800 [1:05:00<2:47:52, 16.07s/it]
+
+{'loss': 0.0478, 'grad_norm': 15.310420989990234, 'learning_rate': 4.354166666666667e-05, 'num_tokens': 3447033.0, 'completions/mean_length': 18.21875, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.3125, 'completions/mean_terminated_length': 11.954545974731445, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 21.0, 'rewards/format_valid/mean': 0.484375, 'rewards/format_valid/std': 0.6896045207977295, 'rewards/action_legal/mean': -0.0625, 'rewards/action_legal/std': 0.24593468010425568, 'rewards/env_reward/mean': 0.47896432876586914, 'rewards/env_reward/std': 1.10792875289917, 'rewards/belief_accuracy/mean': -0.08382219076156616, 'rewards/belief_accuracy/std': 0.07211460173130035, 'reward': 0.4880737364292145, 'reward_std': 0.6930756568908691, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.1938250660896301, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.35}
+
+
+ 22%|██▏       | 174/800 [1:05:16<2:47:23, 16.04s/it]
+
+
+                                                     
+
+
+
+ 22%|██▏       | 174/800 [1:05:16<2:47:23, 16.04s/it]
+
+
+ 22%|██▏       | 175/800 [1:05:32<2:46:56, 16.03s/it]
+
+
+                                                     
+
+
+{'loss': 0.0598, 'grad_norm': 0.6213586330413818, 'learning_rate': 4.3472222222222225e-05, 'num_tokens': 3466860.0, 'completions/mean_length': 19.59375, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.25, 'completions/mean_terminated_length': 15.458333969116211, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 29.0, 'rewards/format_valid/mean': 0.515625, 'rewards/format_valid/std': 0.0883883461356163, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.7535071969032288, 'rewards/env_reward/std': 0.7110505700111389, 'rewards/belief_accuracy/mean': -0.09953412413597107, 'rewards/belief_accuracy/std': 0.00263538327999413, 'reward': 0.8574397563934326, 'reward_std': 0.026050161570310593, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.4946843981742859, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.35}
+
+
+ 22%|██▏       | 175/800 [1:05:32<2:46:56, 16.03s/it]
+
+
+ 22%|██▏       | 176/800 [1:05:48<2:46:34, 16.02s/it]
+
+
+                                                     
+
+
+{'loss': 0.043, 'grad_norm': 6.547880172729492, 'learning_rate': 4.340277777777778e-05, 'num_tokens': 3486736.0, 'completions/mean_length': 21.125, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.375, 'completions/mean_terminated_length': 14.600000381469727, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 23.0, 'rewards/format_valid/mean': 0.265625, 'rewards/format_valid/std': 0.7403614521026611, 'rewards/action_legal/mean': -0.09375, 'rewards/action_legal/std': 0.2961445748806, 'rewards/env_reward/mean': 0.33916449546813965, 'rewards/env_reward/std': 1.2105263471603394, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': 0.2173403799533844, 'reward_std': 0.7407374382019043, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.075773149728775, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.35}
+
+
+ 22%|██▏       | 176/800 [1:05:48<2:46:34, 16.02s/it]
+
+
+ 22%|██▏       | 177/800 [1:06:03<2:46:10, 16.00s/it]
+
+
+                                                     
+
+
+{'loss': 0.0465, 'grad_norm': 7.715331554412842, 'learning_rate': 4.3333333333333334e-05, 'num_tokens': 3506516.0, 'completions/mean_length': 18.125, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.15625, 'completions/mean_terminated_length': 15.55555534362793, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': 0.484375, 'rewards/format_valid/std': 0.48334142565727234, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.8826240301132202, 'rewards/env_reward/std': 0.9095532894134521, 'rewards/belief_accuracy/mean': -0.10375553369522095, 'rewards/belief_accuracy/std': 0.01646575704216957, 'reward': 1.0353256464004517, 'reward_std': 0.5840547680854797, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.1633504331111908, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.35}
+
+
+ 22%|██▏       | 177/800 [1:06:03<2:46:10, 16.00s/it]
+
+
+ 22%|██▏       | 178/800 [1:06:19<2:45:55, 16.01s/it]
+
+
+                                                     
+
+
+{'loss': 0.0443, 'grad_norm': 8.787858963012695, 'learning_rate': 4.3263888888888895e-05, 'num_tokens': 3526346.0, 'completions/mean_length': 19.6875, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.25, 'completions/mean_terminated_length': 15.583333969116211, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.46875, 'rewards/format_valid/std': 0.4741290807723999, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.6897691488265991, 'rewards/env_reward/std': 0.7222986221313477, 'rewards/belief_accuracy/mean': -0.10117440670728683, 'rewards/belief_accuracy/std': 0.0064028408378362656, 'reward': 0.7530053853988647, 'reward_std': 0.5583629608154297, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.1064254641532898, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.36}
+
+
+ 22%|██▏       | 178/800 [1:06:19<2:45:55, 16.01s/it]
+
+{'loss': 0.0588, 'grad_norm': 5.969128131866455, 'learning_rate': 4.319444444444445e-05, 'num_tokens': 3546039.0, 'completions/mean_length': 15.40625, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.125, 'completions/mean_terminated_length': 13.035715103149414, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 21.0, 'rewards/format_valid/mean': 0.546875, 'rewards/format_valid/std': 0.1480722874403, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.6605232357978821, 'rewards/env_reward/std': 0.6082080602645874, 'rewards/belief_accuracy/mean': -0.10268519073724747, 'rewards/belief_accuracy/std': 0.012674203142523766, 'reward': 0.7100729942321777, 'reward_std': 0.02144826389849186, 'frac_reward_zero_std': 0.5, 'completion_length': 32.0, 'kl': 1.4695490598678589, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.36}
+
+
+ 22%|██▏       | 179/800 [1:06:35<2:45:39, 16.01s/it]
+
+
+                                                     
+
+
+
+ 22%|██▏       | 179/800 [1:06:35<2:45:39, 16.01s/it]
+
+
+ 22%|██▎       | 180/800 [1:06:54<2:52:28, 16.69s/it]
+
+
+                                                     
+
+
+{'loss': 0.0657, 'grad_norm': 3.8499722480773926, 'learning_rate': 4.3125000000000005e-05, 'num_tokens': 3565307.0, 'completions/mean_length': 16.125, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.15625, 'completions/mean_terminated_length': 13.185185432434082, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 28.0, 'rewards/format_valid/mean': 0.796875, 'rewards/format_valid/std': 0.55152827501297, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.5543315410614014, 'rewards/env_reward/std': 0.6920597553253174, 'rewards/belief_accuracy/mean': -0.04466632381081581, 'rewards/belief_accuracy/std': 0.07538499683141708, 'reward': 0.7357796430587769, 'reward_std': 0.6875156164169312, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.6429007053375244, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.36}
+
+
+ 22%|██▎       | 180/800 [1:06:54<2:52:28, 16.69s/it]
+
+{'loss': 0.0578, 'grad_norm': 1.5590953826904297, 'learning_rate': 4.305555555555556e-05, 'num_tokens': 3585055.0, 'completions/mean_length': 17.125, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.15625, 'completions/mean_terminated_length': 14.370370864868164, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.671875, 'rewards/format_valid/std': 0.24127934873104095, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.7588062882423401, 'rewards/env_reward/std': 0.5688477754592896, 'rewards/belief_accuracy/mean': -0.08112707734107971, 'rewards/belief_accuracy/std': 0.051890209317207336, 'reward': 0.9284218549728394, 'reward_std': 0.09908436238765717, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.444637417793274, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.36}
+
+
+ 23%|██▎       | 181/800 [1:07:10<2:49:58, 16.48s/it]
+
+
+                                                     
+
+
+
+ 23%|██▎       | 181/800 [1:07:10<2:49:58, 16.48s/it]
+
+
+ 23%|██▎       | 182/800 [1:07:28<2:55:23, 17.03s/it]
+
+
+                                                     
+
+
+{'loss': 0.0564, 'grad_norm': 27.91185188293457, 'learning_rate': 4.2986111111111114e-05, 'num_tokens': 3603951.0, 'completions/mean_length': 18.5, 'completions/min_length': 11.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.1875, 'completions/mean_terminated_length': 15.384615898132324, 'completions/min_terminated_length': 11.0, 'completions/max_terminated_length': 29.0, 'rewards/format_valid/mean': 0.640625, 'rewards/format_valid/std': 0.7321463227272034, 'rewards/action_legal/mean': -0.0625, 'rewards/action_legal/std': 0.24593468010425568, 'rewards/env_reward/mean': 0.6183497905731201, 'rewards/env_reward/std': 1.1782550811767578, 'rewards/belief_accuracy/mean': -0.04769120365381241, 'rewards/belief_accuracy/std': 0.05115366354584694, 'reward': 0.8133573532104492, 'reward_std': 0.9961608052253723, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.4109742641448975, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.36}
+
+
+ 23%|██▎       | 182/800 [1:07:28<2:55:23, 17.03s/it]
+
+{'loss': 0.0648, 'grad_norm': 1.3455158472061157, 'learning_rate': 4.291666666666667e-05, 'num_tokens': 3623708.0, 'completions/mean_length': 17.40625, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.15625, 'completions/mean_terminated_length': 14.703703880310059, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 27.0, 'rewards/format_valid/mean': 0.5, 'rewards/format_valid/std': 0.0, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.5056562423706055, 'rewards/env_reward/std': 0.33424797654151917, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': 0.4834843575954437, 'reward_std': 0.0, 'frac_reward_zero_std': 1.0, 'completion_length': 32.0, 'kl': 1.6201136410236359, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.37}
+
+
+ 23%|██▎       | 183/800 [1:07:44<2:51:55, 16.72s/it]
+
+
+                                                     
+
+
+
+ 23%|██▎       | 183/800 [1:07:44<2:51:55, 16.72s/it]
+
+{'loss': 0.0551, 'grad_norm': 3.1338021755218506, 'learning_rate': 4.284722222222222e-05, 'num_tokens': 3642952.0, 'completions/mean_length': 15.375, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.15625, 'completions/mean_terminated_length': 12.296296119689941, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 23.0, 'rewards/format_valid/mean': 0.625, 'rewards/format_valid/std': 0.2199706733226776, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.42772161960601807, 'rewards/env_reward/std': 0.3833340108394623, 'rewards/belief_accuracy/mean': -0.07500000298023224, 'rewards/belief_accuracy/std': 0.04399413242936134, 'reward': 0.4478324055671692, 'reward_std': 0.04361426830291748, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.3781002461910248, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.37}
+
+
+ 23%|██▎       | 184/800 [1:08:02<2:56:31, 17.19s/it]
+
+
+                                                     
+
+
+
+ 23%|██▎       | 184/800 [1:08:02<2:56:31, 17.19s/it]
+
+
+ 23%|██▎       | 185/800 [1:08:18<2:52:35, 16.84s/it]
+
+
+                                                     
+
+
+{'loss': 0.0712, 'grad_norm': 4.9064531326293945, 'learning_rate': 4.277777777777778e-05, 'num_tokens': 3662591.0, 'completions/mean_length': 13.71875, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.03125, 'completions/mean_terminated_length': 13.129032135009766, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 22.0, 'rewards/format_valid/mean': 0.53125, 'rewards/format_valid/std': 0.12296734005212784, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.1845788061618805, 'rewards/env_reward/std': 0.5008557438850403, 'rewards/belief_accuracy/mean': -0.09003403782844543, 'rewards/belief_accuracy/std': 0.039820052683353424, 'reward': 0.0333285927772522, 'reward_std': 0.0889834463596344, 'frac_reward_zero_std': 0.5, 'completion_length': 32.0, 'kl': 1.778859943151474, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.37}
+
+
+ 23%|██▎       | 185/800 [1:08:18<2:52:35, 16.84s/it]
+
+{'loss': 0.0546, 'grad_norm': 7.354944705963135, 'learning_rate': 4.270833333333333e-05, 'num_tokens': 3682302.0, 'completions/mean_length': 15.96875, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.0625, 'completions/mean_terminated_length': 14.90000057220459, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': 0.640625, 'rewards/format_valid/std': 0.22840170562267303, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.5258901119232178, 'rewards/env_reward/std': 0.2250821888446808, 'rewards/belief_accuracy/mean': -0.10883036255836487, 'rewards/belief_accuracy/std': 0.04511723294854164, 'reward': 0.4943752884864807, 'reward_std': 0.08972756564617157, 'frac_reward_zero_std': 0.5, 'completion_length': 32.0, 'kl': 1.3656180500984192, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.37}
+
+
+ 23%|██▎       | 186/800 [1:08:34<2:49:37, 16.58s/it]
+
+
+                                                     
+
+
+
+ 23%|██▎       | 186/800 [1:08:34<2:49:37, 16.58s/it]
+
+{'loss': 0.065, 'grad_norm': 2.4717726707458496, 'learning_rate': 4.263888888888889e-05, 'num_tokens': 3702057.0, 'completions/mean_length': 17.34375, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.21875, 'completions/mean_terminated_length': 13.239999771118164, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 24.0, 'rewards/format_valid/mean': 0.546875, 'rewards/format_valid/std': 0.1480722874403, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.10037755221128464, 'rewards/env_reward/std': 0.11899495124816895, 'rewards/belief_accuracy/mean': -0.10130541771650314, 'rewards/belief_accuracy/std': 0.02356705814599991, 'reward': -0.12600615620613098, 'reward_std': 0.03668661043047905, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.6259745657444, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.37}
+
+
+ 23%|██▎       | 187/800 [1:08:50<2:47:33, 16.40s/it]
+
+
+                                                     
+
+
+
+ 23%|██▎       | 187/800 [1:08:50<2:47:33, 16.40s/it]
+
+
+ 24%|██▎       | 188/800 [1:09:09<2:53:10, 16.98s/it]
+
+
+                                                     
+
+
+{'loss': 0.0538, 'grad_norm': 3.2334837913513184, 'learning_rate': 4.256944444444445e-05, 'num_tokens': 3721253.0, 'completions/mean_length': 13.875, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.03125, 'completions/mean_terminated_length': 13.290322303771973, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.625, 'rewards/format_valid/std': 0.2199706733226776, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.6963561773300171, 'rewards/env_reward/std': 0.3888089060783386, 'rewards/belief_accuracy/mean': -0.07500000298023224, 'rewards/belief_accuracy/std': 0.04399413242936134, 'reward': 0.850784182548523, 'reward_std': 0.04361426457762718, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.345284342765808, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.38}
+
+
+ 24%|██▎       | 188/800 [1:09:09<2:53:10, 16.98s/it]
+
+
+ 24%|██▎       | 189/800 [1:09:27<2:56:57, 17.38s/it]
+
+
+                                                     
+
+
+{'loss': 0.0494, 'grad_norm': 6.991729259490967, 'learning_rate': 4.25e-05, 'num_tokens': 3740496.0, 'completions/mean_length': 15.34375, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.15625, 'completions/mean_terminated_length': 12.259259223937988, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 27.0, 'rewards/format_valid/mean': 0.625, 'rewards/format_valid/std': 0.2199706733226776, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.6645544171333313, 'rewards/env_reward/std': 0.3397876024246216, 'rewards/belief_accuracy/mean': -0.07500000298023224, 'rewards/belief_accuracy/std': 0.04399413242936134, 'reward': 0.8030815720558167, 'reward_std': 0.11756756156682968, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.2358044981956482, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.38}
+
+
+ 24%|██▎       | 189/800 [1:09:27<2:56:57, 17.38s/it]
+
+
+ 24%|██▍       | 190/800 [1:09:43<2:52:27, 16.96s/it]
+
+
+                                                     
+
+
+{'loss': 0.0539, 'grad_norm': 9.369535446166992, 'learning_rate': 4.243055555555556e-05, 'num_tokens': 3760225.0, 'completions/mean_length': 16.53125, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.0625, 'completions/mean_terminated_length': 15.500000953674316, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': 0.640625, 'rewards/format_valid/std': 0.22840170562267303, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.7316009998321533, 'rewards/env_reward/std': 0.5933727622032166, 'rewards/belief_accuracy/mean': -0.06604166328907013, 'rewards/belief_accuracy/std': 0.070061095058918, 'reward': 0.9313077330589294, 'reward_std': 0.1580241322517395, 'frac_reward_zero_std': 0.5, 'completion_length': 32.0, 'kl': 1.347444862127304, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.38}
+
+
+ 24%|██▍       | 190/800 [1:09:43<2:52:27, 16.96s/it]
+
+{'loss': 0.0473, 'grad_norm': 1.727048397064209, 'learning_rate': 4.236111111111111e-05, 'num_tokens': 3779415.0, 'completions/mean_length': 13.6875, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.09375, 'completions/mean_terminated_length': 11.793103218078613, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 25.0, 'rewards/format_valid/mean': 0.625, 'rewards/format_valid/std': 0.2199706733226776, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.7651158571243286, 'rewards/env_reward/std': 0.6568248867988586, 'rewards/belief_accuracy/mean': -0.07500000298023224, 'rewards/belief_accuracy/std': 0.04399413242936134, 'reward': 0.9539238214492798, 'reward_std': 0.026038678362965584, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.1826866269111633, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.38}
+
+
+ 24%|██▍       | 191/800 [1:10:01<2:56:15, 17.37s/it]
+
+
+                                                     
+
+
+
+ 24%|██▍       | 191/800 [1:10:01<2:56:15, 17.37s/it]
+
+
+ 24%|██▍       | 192/800 [1:10:17<2:51:47, 16.95s/it]
+
+
+                                                     
+
+
+{'loss': 0.0526, 'grad_norm': 12.252972602844238, 'learning_rate': 4.229166666666667e-05, 'num_tokens': 3799157.0, 'completions/mean_length': 16.9375, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.21875, 'completions/mean_terminated_length': 12.719999313354492, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.453125, 'rewards/format_valid/std': 0.4641915261745453, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.33155351877212524, 'rewards/env_reward/std': 0.6488659381866455, 'rewards/belief_accuracy/mean': -0.0989583358168602, 'rewards/belief_accuracy/std': 0.013345088809728622, 'reward': 0.22154900431632996, 'reward_std': 0.4587080478668213, 'frac_reward_zero_std': 0.5, 'completion_length': 32.0, 'kl': 1.3160167336463928, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.38}
+
+
+ 24%|██▍       | 192/800 [1:10:17<2:51:47, 16.95s/it]
+
+{'loss': 0.0468, 'grad_norm': 0.8445842266082764, 'learning_rate': 4.222222222222222e-05, 'num_tokens': 3818448.0, 'completions/mean_length': 16.84375, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.1875, 'completions/mean_terminated_length': 13.34615421295166, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 26.0, 'rewards/format_valid/mean': 0.625, 'rewards/format_valid/std': 0.2199706733226776, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.570223867893219, 'rewards/env_reward/std': 0.36964988708496094, 'rewards/belief_accuracy/mean': -0.07500000298023224, 'rewards/belief_accuracy/std': 0.04399413242936134, 'reward': 0.661585807800293, 'reward_std': 0.019887370988726616, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.1687933057546616, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.39}
+
+
+ 24%|██▍       | 193/800 [1:10:36<2:55:35, 17.36s/it]
+
+
+                                                     
+
+
+
+ 24%|██▍       | 193/800 [1:10:36<2:55:35, 17.36s/it]
+
+{'loss': 0.0446, 'grad_norm': 5.747774124145508, 'learning_rate': 4.215277777777778e-05, 'num_tokens': 3838098.0, 'completions/mean_length': 14.0625, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.15625, 'completions/mean_terminated_length': 10.740740776062012, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 27.0, 'rewards/format_valid/mean': 0.515625, 'rewards/format_valid/std': 0.0883883461356163, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.6394833922386169, 'rewards/env_reward/std': 0.49910008907318115, 'rewards/belief_accuracy/mean': -0.09385417401790619, 'rewards/belief_accuracy/std': 0.03476608172059059, 'reward': 0.7034438252449036, 'reward_std': 0.07424620538949966, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.1156142055988312, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.39}
+
+
+ 24%|██▍       | 194/800 [1:10:52<2:51:12, 16.95s/it]
+
+
+                                                     
+
+
+
+ 24%|██▍       | 194/800 [1:10:52<2:51:12, 16.95s/it]
+
+
+ 24%|██▍       | 195/800 [1:11:08<2:48:08, 16.68s/it]
+
+
+                                                     
+
+
+{'loss': 0.0515, 'grad_norm': 1.714820146560669, 'learning_rate': 4.208333333333334e-05, 'num_tokens': 3857835.0, 'completions/mean_length': 16.78125, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.1875, 'completions/mean_terminated_length': 13.269230842590332, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 25.0, 'rewards/format_valid/mean': 0.515625, 'rewards/format_valid/std': 0.0883883461356163, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.40318048000335693, 'rewards/env_reward/std': 0.43285053968429565, 'rewards/belief_accuracy/mean': -0.10092592239379883, 'rewards/belief_accuracy/std': 0.005237827077507973, 'reward': 0.32777416706085205, 'reward_std': 0.005647021345794201, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.2879925668239594, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.39}
+
+
+ 24%|██▍       | 195/800 [1:11:08<2:48:08, 16.68s/it]
+
+{'loss': 0.0686, 'grad_norm': 9.99744987487793, 'learning_rate': 4.201388888888889e-05, 'num_tokens': 3877472.0, 'completions/mean_length': 13.65625, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.0625, 'completions/mean_terminated_length': 12.433334350585938, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': 0.515625, 'rewards/format_valid/std': 0.0883883461356163, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.44838035106658936, 'rewards/env_reward/std': 0.3025989532470703, 'rewards/belief_accuracy/mean': -0.10093750059604645, 'rewards/belief_accuracy/std': 0.005303299520164728, 'reward': 0.395539253950119, 'reward_std': 0.005745241418480873, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.7159236371517181, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.39}
+
+
+ 24%|██▍       | 196/800 [1:11:24<2:45:52, 16.48s/it]
+
+
+                                                     
+
+
+
+ 24%|██▍       | 196/800 [1:11:24<2:45:52, 16.48s/it]
+
+
+ 25%|██▍       | 197/800 [1:11:40<2:44:04, 16.33s/it]
+
+
+                                                     
+
+
+{'loss': 0.0503, 'grad_norm': 3.2998619079589844, 'learning_rate': 4.194444444444445e-05, 'num_tokens': 3897198.0, 'completions/mean_length': 16.4375, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.15625, 'completions/mean_terminated_length': 13.55555534362793, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.5, 'rewards/format_valid/std': 0.0, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.8880956172943115, 'rewards/env_reward/std': 0.38598909974098206, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': 1.0571434497833252, 'reward_std': 0.0, 'frac_reward_zero_std': 1.0, 'completion_length': 32.0, 'kl': 1.2571548223495483, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.39}
+
+
+ 25%|██▍       | 197/800 [1:11:40<2:44:04, 16.33s/it]
+
+{'loss': 0.0592, 'grad_norm': 0.2773527503013611, 'learning_rate': 4.1875e-05, 'num_tokens': 3916906.0, 'completions/mean_length': 15.875, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.1875, 'completions/mean_terminated_length': 12.153846740722656, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 25.0, 'rewards/format_valid/mean': 0.5, 'rewards/format_valid/std': 0.0, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.4838310480117798, 'rewards/env_reward/std': 0.5967168807983398, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': 0.4507465660572052, 'reward_std': 0.0, 'frac_reward_zero_std': 1.0, 'completion_length': 32.0, 'kl': 1.4807668030261993, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.4}
+
+
+ 25%|██▍       | 198/800 [1:11:56<2:42:48, 16.23s/it]
+
+
+                                                     
+
+
+
+ 25%|██▍       | 198/800 [1:11:56<2:42:48, 16.23s/it]
+
+
+ 25%|██▍       | 199/800 [1:12:12<2:41:54, 16.16s/it]
+
+
+                                                     
+
+
+{'loss': 0.0551, 'grad_norm': 1.857956051826477, 'learning_rate': 4.1805555555555556e-05, 'num_tokens': 3936584.0, 'completions/mean_length': 14.9375, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.15625, 'completions/mean_terminated_length': 11.777777671813965, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 25.0, 'rewards/format_valid/mean': 0.5, 'rewards/format_valid/std': 0.0, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.8568747043609619, 'rewards/env_reward/std': 0.30477213859558105, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': 1.0103120803833008, 'reward_std': 0.0, 'frac_reward_zero_std': 1.0, 'completion_length': 32.0, 'kl': 1.3785418570041656, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.4}
+
+
+ 25%|██▍       | 199/800 [1:12:12<2:41:54, 16.16s/it]
+
+
+ 25%|██▌       | 200/800 [1:12:30<2:48:02, 16.80s/it]
+
+
+                                                     
+
+
+{'loss': 0.0349, 'grad_norm': 15.719167709350586, 'learning_rate': 4.173611111111112e-05, 'num_tokens': 3955852.0, 'completions/mean_length': 16.125, 'completions/min_length': 9.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.1875, 'completions/mean_terminated_length': 12.461539268493652, 'completions/min_terminated_length': 9.0, 'completions/max_terminated_length': 23.0, 'rewards/format_valid/mean': 0.875, 'rewards/format_valid/std': 0.2199706733226776, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.7512513399124146, 'rewards/env_reward/std': 0.6829286813735962, 'rewards/belief_accuracy/mean': -0.042389631271362305, 'rewards/belief_accuracy/std': 0.06569451838731766, 'reward': 1.043458104133606, 'reward_std': 0.2043767124414444, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 0.8725590407848358, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.4}
+
+
+ 25%|██▌       | 200/800 [1:12:30<2:48:02, 16.80s/it]
+
+
+ 25%|██▌       | 201/800 [1:12:46<2:45:20, 16.56s/it]
+
+
+                                                     
+
+
+{'loss': 0.052, 'grad_norm': 6.8372273445129395, 'learning_rate': 4.166666666666667e-05, 'num_tokens': 3975545.0, 'completions/mean_length': 15.40625, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.0625, 'completions/mean_terminated_length': 14.30000114440918, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.625, 'rewards/format_valid/std': 0.2199706733226776, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.7056492567062378, 'rewards/env_reward/std': 0.7769469618797302, 'rewards/belief_accuracy/mean': -0.1514628678560257, 'rewards/belief_accuracy/std': 0.09509190917015076, 'reward': 0.6353352665901184, 'reward_std': 0.0507856160402298, 'frac_reward_zero_std': 0.5, 'completion_length': 32.0, 'kl': 1.3000997602939606, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.4}
+
+
+ 25%|██▌       | 201/800 [1:12:46<2:45:20, 16.56s/it]
+
+
+ 25%|██▌       | 202/800 [1:13:02<2:43:24, 16.40s/it]
+
+
+                                                     
+
+
+{'loss': 0.0507, 'grad_norm': 0.29732486605644226, 'learning_rate': 4.159722222222223e-05, 'num_tokens': 3995180.0, 'completions/mean_length': 13.59375, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.0625, 'completions/mean_terminated_length': 12.366667747497559, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 28.0, 'rewards/format_valid/mean': 0.5, 'rewards/format_valid/std': 0.0, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.6812490820884705, 'rewards/env_reward/std': 0.5589019060134888, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': 0.7468736171722412, 'reward_std': 0.0, 'frac_reward_zero_std': 1.0, 'completion_length': 32.0, 'kl': 1.266777753829956, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.4}
+
+
+ 25%|██▌       | 202/800 [1:13:02<2:43:24, 16.40s/it]
+
+{'loss': 0.0387, 'grad_norm': 2.056792736053467, 'learning_rate': 4.152777777777778e-05, 'num_tokens': 4014381.0, 'completions/mean_length': 14.03125, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.0625, 'completions/mean_terminated_length': 12.833333969116211, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 29.0, 'rewards/format_valid/mean': 0.53125, 'rewards/format_valid/std': 0.507007360458374, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.5290133357048035, 'rewards/env_reward/std': 0.6629040837287903, 'rewards/belief_accuracy/mean': -0.078125, 'rewards/belief_accuracy/std': 0.04200134426355362, 'reward': 0.5841450691223145, 'reward_std': 0.5143226385116577, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 0.9673857390880585, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.41}
+
+
+ 25%|██▌       | 203/800 [1:13:20<2:48:49, 16.97s/it]
+
+
+                                                     
+
+
+
+ 25%|██▌       | 203/800 [1:13:20<2:48:49, 16.97s/it]
+
+
+ 26%|██▌       | 204/800 [1:13:36<2:45:37, 16.67s/it]
+
+
+                                                     
+
+
+{'loss': 0.0498, 'grad_norm': 0.1867222636938095, 'learning_rate': 4.1458333333333336e-05, 'num_tokens': 4034046.0, 'completions/mean_length': 14.53125, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.125, 'completions/mean_terminated_length': 12.035715103149414, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 23.0, 'rewards/format_valid/mean': 0.5, 'rewards/format_valid/std': 0.0, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.2954077124595642, 'rewards/env_reward/std': 0.14831718802452087, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': 0.16811153292655945, 'reward_std': 0.0, 'frac_reward_zero_std': 1.0, 'completion_length': 32.0, 'kl': 1.2444508373737335, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.41}
+
+
+ 26%|██▌       | 204/800 [1:13:36<2:45:37, 16.67s/it]
+
+
+ 26%|██▌       | 205/800 [1:13:52<2:43:21, 16.47s/it]
+
+
+                                                     
+
+
+{'loss': 0.0548, 'grad_norm': 19.916845321655273, 'learning_rate': 4.138888888888889e-05, 'num_tokens': 4053755.0, 'completions/mean_length': 15.90625, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.1875, 'completions/mean_terminated_length': 12.19230842590332, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.515625, 'rewards/format_valid/std': 0.0883883461356163, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 1.1051976680755615, 'rewards/env_reward/std': 0.16549694538116455, 'rewards/belief_accuracy/mean': -0.09745369851589203, 'rewards/belief_accuracy/std': 0.014404026791453362, 'reward': 1.391216516494751, 'reward_std': 0.02381575107574463, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.3709315359592438, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.41}
+
+
+ 26%|██▌       | 205/800 [1:13:52<2:43:21, 16.47s/it]
+
+
+ 26%|██▌       | 206/800 [1:14:08<2:41:36, 16.32s/it]
+
+
+                                                     
+
+
+{'loss': 0.0606, 'grad_norm': 2.390664577484131, 'learning_rate': 4.1319444444444445e-05, 'num_tokens': 4073456.0, 'completions/mean_length': 15.65625, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.15625, 'completions/mean_terminated_length': 12.629630088806152, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 28.0, 'rewards/format_valid/mean': 0.5, 'rewards/format_valid/std': 0.0, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.4062948226928711, 'rewards/env_reward/std': 0.2933061718940735, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': 0.3344421982765198, 'reward_std': 0.0, 'frac_reward_zero_std': 1.0, 'completion_length': 32.0, 'kl': 1.514962986111641, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.41}
+
+
+ 26%|██▌       | 206/800 [1:14:08<2:41:36, 16.32s/it]
+
+
+ 26%|██▌       | 207/800 [1:14:24<2:40:12, 16.21s/it]
+
+
+                                                     
+
+
+{'loss': 0.0559, 'grad_norm': 0.9200486540794373, 'learning_rate': 4.125e-05, 'num_tokens': 4093112.0, 'completions/mean_length': 14.25, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.125, 'completions/mean_terminated_length': 11.714285850524902, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.421875, 'rewards/format_valid/std': 0.4419417381286621, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.49185124039649963, 'rewards/env_reward/std': 0.8825413584709167, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': 0.45730817317962646, 'reward_std': 0.5390381217002869, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.396568775177002, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.41}
+
+
+ 26%|██▌       | 207/800 [1:14:24<2:40:12, 16.21s/it]
+
+
+ 26%|██▌       | 208/800 [1:14:42<2:46:08, 16.84s/it]
+
+
+                                                     
+
+
+{'loss': 0.0415, 'grad_norm': 6.145766258239746, 'learning_rate': 4.1180555555555554e-05, 'num_tokens': 4111915.0, 'completions/mean_length': 15.59375, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.15625, 'completions/mean_terminated_length': 12.55555534362793, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 29.0, 'rewards/format_valid/mean': 0.75, 'rewards/format_valid/std': 0.2540002465248108, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.518890380859375, 'rewards/env_reward/std': 0.28433749079704285, 'rewards/belief_accuracy/mean': -0.05052083358168602, 'rewards/belief_accuracy/std': 0.05035436525940895, 'reward': 0.6642730236053467, 'reward_std': 0.15380990505218506, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.037576973438263, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.42}
+
+
+ 26%|██▌       | 208/800 [1:14:42<2:46:08, 16.84s/it]
+
+
+ 26%|██▌       | 209/800 [1:15:01<2:50:17, 17.29s/it]
+
+
+                                                     
+
+
+{'loss': 0.0454, 'grad_norm': 1.5654785633087158, 'learning_rate': 4.111111111111111e-05, 'num_tokens': 4131115.0, 'completions/mean_length': 14.0, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.125, 'completions/mean_terminated_length': 11.428571701049805, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 22.0, 'rewards/format_valid/mean': 0.640625, 'rewards/format_valid/std': 0.22840170562267303, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.3916810154914856, 'rewards/env_reward/std': 0.19149896502494812, 'rewards/belief_accuracy/mean': -0.07476852089166641, 'rewards/belief_accuracy/std': 0.04387768357992172, 'reward': 0.3952471613883972, 'reward_std': 0.024061277508735657, 'frac_reward_zero_std': 0.5, 'completion_length': 32.0, 'kl': 1.1356377303600311, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.42}
+
+
+ 26%|██▌       | 209/800 [1:15:01<2:50:17, 17.29s/it]
+
+
+ 26%|██▋       | 210/800 [1:15:17<2:46:09, 16.90s/it]
+
+
+                                                     
+
+
+{'loss': 0.057, 'grad_norm': 2.324124574661255, 'learning_rate': 4.104166666666667e-05, 'num_tokens': 4150811.0, 'completions/mean_length': 15.5, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.1875, 'completions/mean_terminated_length': 11.69230842590332, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 27.0, 'rewards/format_valid/mean': 0.625, 'rewards/format_valid/std': 0.2199706733226776, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.7758002877235413, 'rewards/env_reward/std': 0.417151540517807, 'rewards/belief_accuracy/mean': -0.12191874533891678, 'rewards/belief_accuracy/std': 0.046972136944532394, 'reward': 0.829194188117981, 'reward_std': 0.05274483188986778, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.424360066652298, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.42}
+
+
+ 26%|██▋       | 210/800 [1:15:17<2:46:09, 16.90s/it]
+
+{'loss': 0.0448, 'grad_norm': 10.221343040466309, 'learning_rate': 4.0972222222222225e-05, 'num_tokens': 4170146.0, 'completions/mean_length': 18.21875, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.25, 'completions/mean_terminated_length': 13.625, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 27.0, 'rewards/format_valid/mean': 0.640625, 'rewards/format_valid/std': 0.22840170562267303, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.45686447620391846, 'rewards/env_reward/std': 0.3069411814212799, 'rewards/belief_accuracy/mean': -0.07187500596046448, 'rewards/belief_accuracy/std': 0.04568034037947655, 'reward': 0.5017029643058777, 'reward_std': 0.02872621640563011, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.120680257678032, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.42}
+
+
+ 26%|██▋       | 211/800 [1:15:35<2:50:05, 17.33s/it]
+
+
+                                                     
+
+
+
+ 26%|██▋       | 211/800 [1:15:35<2:50:05, 17.33s/it]
+
+
+ 26%|██▋       | 212/800 [1:15:53<2:52:42, 17.62s/it]
+
+
+                                                     
+
+
+{'loss': 0.0397, 'grad_norm': 1.2832083702087402, 'learning_rate': 4.090277777777778e-05, 'num_tokens': 4189434.0, 'completions/mean_length': 16.75, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.25, 'completions/mean_terminated_length': 11.666666984558105, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.640625, 'rewards/format_valid/std': 0.22840170562267303, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.6780300140380859, 'rewards/env_reward/std': 0.4279901683330536, 'rewards/belief_accuracy/mean': -0.07447917014360428, 'rewards/belief_accuracy/std': 0.04378678649663925, 'reward': 0.8256387710571289, 'reward_std': 0.043144673109054565, 'frac_reward_zero_std': 0.5, 'completion_length': 32.0, 'kl': 0.9913371652364731, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.42}
+
+
+ 26%|██▋       | 212/800 [1:15:53<2:52:42, 17.62s/it]
+
+
+ 27%|██▋       | 213/800 [1:16:12<2:54:30, 17.84s/it]
+
+
+                                                     
+
+
+{'loss': 0.0474, 'grad_norm': 2.684176445007324, 'learning_rate': 4.0833333333333334e-05, 'num_tokens': 4208698.0, 'completions/mean_length': 16.0, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.21875, 'completions/mean_terminated_length': 11.519999504089355, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 29.0, 'rewards/format_valid/mean': 0.609375, 'rewards/format_valid/std': 0.2100067138671875, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.22834785282611847, 'rewards/env_reward/std': 0.42787954211235046, 'rewards/belief_accuracy/mean': -0.078125, 'rewards/belief_accuracy/std': 0.04200134426355362, 'reward': 0.13861548900604248, 'reward_std': 0.10686702281236649, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.184818834066391, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.43}
+
+
+ 27%|██▋       | 213/800 [1:16:12<2:54:30, 17.84s/it]
+
+{'loss': 0.051, 'grad_norm': 2.677314043045044, 'learning_rate': 4.076388888888889e-05, 'num_tokens': 4228524.0, 'completions/mean_length': 19.5625, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.3125, 'completions/mean_terminated_length': 13.909090995788574, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 27.0, 'rewards/format_valid/mean': 0.421875, 'rewards/format_valid/std': 0.4419417381286621, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.1121930181980133, 'rewards/env_reward/std': 0.7171587944030762, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': -0.1121792197227478, 'reward_std': 0.4342537522315979, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.274095743894577, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.43}
+
+
+ 27%|██▋       | 214/800 [1:16:28<2:48:48, 17.28s/it]
+
+
+                                                     
+
+
+
+ 27%|██▋       | 214/800 [1:16:28<2:48:48, 17.28s/it]
+
+{'loss': 0.0442, 'grad_norm': 4.336506366729736, 'learning_rate': 4.0694444444444444e-05, 'num_tokens': 4248366.0, 'completions/mean_length': 20.0625, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.28125, 'completions/mean_terminated_length': 15.391304969787598, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 29.0, 'rewards/format_valid/mean': 0.03125, 'rewards/format_valid/std': 0.9913944005966187, 'rewards/action_legal/mean': -0.1875, 'rewards/action_legal/std': 0.3965577781200409, 'rewards/env_reward/mean': -0.19335293769836426, 'rewards/env_reward/std': 1.5103976726531982, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': -0.5978419184684753, 'reward_std': 0.7300028800964355, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.1055477857589722, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.43}
+
+
+ 27%|██▋       | 215/800 [1:16:44<2:44:40, 16.89s/it]
+
+
+                                                     
+
+
+
+ 27%|██▋       | 215/800 [1:16:44<2:44:40, 16.89s/it]
+
+{'loss': 0.0427, 'grad_norm': 8.889293670654297, 'learning_rate': 4.0625000000000005e-05, 'num_tokens': 4268114.0, 'completions/mean_length': 17.125, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.1875, 'completions/mean_terminated_length': 13.69230842590332, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 26.0, 'rewards/format_valid/mean': 0.515625, 'rewards/format_valid/std': 0.0883883461356163, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.4521803855895996, 'rewards/env_reward/std': 0.2416244000196457, 'rewards/belief_accuracy/mean': -0.09758079051971436, 'rewards/belief_accuracy/std': 0.013685100711882114, 'reward': 0.41130945086479187, 'reward_std': 0.022737354040145874, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.0670965313911438, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.43}
+
+
+ 27%|██▋       | 216/800 [1:17:00<2:41:46, 16.62s/it]
+
+
+                                                     
+
+
+
+ 27%|██▋       | 216/800 [1:17:00<2:41:46, 16.62s/it]
+
+
+ 27%|██▋       | 217/800 [1:17:16<2:39:32, 16.42s/it]
+
+
+                                                     
+
+
+{'loss': 0.0454, 'grad_norm': 52.85539627075195, 'learning_rate': 4.055555555555556e-05, 'num_tokens': 4287944.0, 'completions/mean_length': 19.6875, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.25, 'completions/mean_terminated_length': 15.583333969116211, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 26.0, 'rewards/format_valid/mean': 0.234375, 'rewards/format_valid/std': 0.9998739361763, 'rewards/action_legal/mean': -0.15625, 'rewards/action_legal/std': 0.3689020276069641, 'rewards/env_reward/mean': 0.4210810661315918, 'rewards/env_reward/std': 1.546980381011963, 'rewards/belief_accuracy/mean': -0.06812499463558197, 'rewards/belief_accuracy/std': 0.06098908185958862, 'reward': 0.4311528205871582, 'reward_std': 1.0996392965316772, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.135423719882965, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.43}
+
+
+ 27%|██▋       | 217/800 [1:17:16<2:39:32, 16.42s/it]
+
+{'loss': 0.0501, 'grad_norm': 10.320279121398926, 'learning_rate': 4.0486111111111114e-05, 'num_tokens': 4307652.0, 'completions/mean_length': 15.875, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.15625, 'completions/mean_terminated_length': 12.88888931274414, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 24.0, 'rewards/format_valid/mean': 0.640625, 'rewards/format_valid/std': 0.22840170562267303, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.40635257959365845, 'rewards/env_reward/std': 0.620490312576294, 'rewards/belief_accuracy/mean': -0.12488628923892975, 'rewards/belief_accuracy/std': 0.05421588942408562, 'reward': 0.26690125465393066, 'reward_std': 0.0767488181591034, 'frac_reward_zero_std': 0.5, 'completion_length': 32.0, 'kl': 1.2529677152633667, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.44}
+
+
+ 27%|██▋       | 218/800 [1:17:32<2:37:58, 16.29s/it]
+
+
+                                                     
+
+
+
+ 27%|██▋       | 218/800 [1:17:32<2:37:58, 16.29s/it]
+
+
+ 27%|██▋       | 219/800 [1:17:50<2:43:37, 16.90s/it]
+
+
+                                                     
+
+
+{'loss': 0.0593, 'grad_norm': 25.136472702026367, 'learning_rate': 4.041666666666667e-05, 'num_tokens': 4326906.0, 'completions/mean_length': 15.6875, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.15625, 'completions/mean_terminated_length': 12.666666984558105, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 27.0, 'rewards/format_valid/mean': 0.65625, 'rewards/format_valid/std': 0.23546454310417175, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.655311107635498, 'rewards/env_reward/std': 0.4426242709159851, 'rewards/belief_accuracy/mean': -0.07260416448116302, 'rewards/belief_accuracy/std': 0.04507051035761833, 'reward': 0.797966718673706, 'reward_std': 0.2623561918735504, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.48312908411026, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.44}
+
+
+ 27%|██▋       | 219/800 [1:17:50<2:43:37, 16.90s/it]
+
+{'loss': 0.0354, 'grad_norm': 28.7308292388916, 'learning_rate': 4.0347222222222223e-05, 'num_tokens': 4346683.0, 'completions/mean_length': 18.03125, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.1875, 'completions/mean_terminated_length': 14.807692527770996, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 24.0, 'rewards/format_valid/mean': 0.765625, 'rewards/format_valid/std': 0.253503680229187, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.5355064868927002, 'rewards/env_reward/std': 0.439134806394577, 'rewards/belief_accuracy/mean': -0.10416555404663086, 'rewards/belief_accuracy/std': 0.0675935223698616, 'reward': 0.5290443301200867, 'reward_std': 0.15863001346588135, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.8857318013906479, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.44}
+
+
+ 28%|██▊       | 220/800 [1:18:06<2:40:46, 16.63s/it]
+
+
+                                                     
+
+
+
+ 28%|██▊       | 220/800 [1:18:06<2:40:46, 16.63s/it]
+
+
+ 28%|██▊       | 221/800 [1:18:22<2:38:37, 16.44s/it]
+
+{'loss': 0.0381, 'grad_norm': 4.578545093536377, 'learning_rate': 4.027777777777778e-05, 'num_tokens': 4366419.0, 'completions/mean_length': 16.75, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.125, 'completions/mean_terminated_length': 14.571429252624512, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.421875, 'rewards/format_valid/std': 0.4419417381286621, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.39418846368789673, 'rewards/env_reward/std': 0.8436952233314514, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': 0.3108140230178833, 'reward_std': 0.4268867075443268, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 0.9517952799797058, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.44}
+
+
+                                                     
+
+
+
+ 28%|██▊       | 221/800 [1:18:22<2:38:37, 16.44s/it]
+
+
+ 28%|██▊       | 222/800 [1:18:38<2:36:59, 16.30s/it]
+
+
+                                                     
+
+
+{'loss': 0.0396, 'grad_norm': 8.620429039001465, 'learning_rate': 4.020833333333334e-05, 'num_tokens': 4386218.0, 'completions/mean_length': 18.71875, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.28125, 'completions/mean_terminated_length': 13.52173900604248, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': -0.109375, 'rewards/format_valid/std': 1.1124976873397827, 'rewards/action_legal/mean': -0.25, 'rewards/action_legal/std': 0.4399413466453552, 'rewards/env_reward/mean': -0.13938742876052856, 'rewards/env_reward/std': 1.7016496658325195, 'rewards/belief_accuracy/mean': -0.10062500834465027, 'rewards/belief_accuracy/std': 0.0035355330910533667, 'reward': -0.5289249420166016, 'reward_std': 0.9563051462173462, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 0.9898501634597778, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.44}
+
+
+ 28%|██▊       | 222/800 [1:18:38<2:36:59, 16.30s/it]
+
+{'loss': 0.0427, 'grad_norm': 1.7900071144104004, 'learning_rate': 4.0138888888888894e-05, 'num_tokens': 4405947.0, 'completions/mean_length': 16.53125, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.1875, 'completions/mean_terminated_length': 12.961539268493652, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 21.0, 'rewards/format_valid/mean': 0.5, 'rewards/format_valid/std': 0.0, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.19219696521759033, 'rewards/env_reward/std': 0.698978066444397, 'rewards/belief_accuracy/mean': -0.10000000149011612, 'rewards/belief_accuracy/std': 0.0, 'reward': 0.013295352458953857, 'reward_std': 0.0, 'frac_reward_zero_std': 1.0, 'completion_length': 32.0, 'kl': 1.0669353157281876, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.45}
+
+
+ 28%|██▊       | 223/800 [1:18:54<2:35:48, 16.20s/it]
+
+
+                                                     
+
+
+
+ 28%|██▊       | 223/800 [1:18:54<2:35:48, 16.20s/it]
+
+
+ 28%|██▊       | 224/800 [1:19:10<2:34:53, 16.14s/it]
+
+
+                                                     
+
+
+{'loss': 0.04, 'grad_norm': 1.6533915996551514, 'learning_rate': 4.006944444444445e-05, 'num_tokens': 4425719.0, 'completions/mean_length': 17.875, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.21875, 'completions/mean_terminated_length': 13.920000076293945, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': 0.53125, 'rewards/format_valid/std': 0.12296734005212784, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.5996853113174438, 'rewards/env_reward/std': 0.46174198389053345, 'rewards/belief_accuracy/mean': -0.09709931910037994, 'rewards/belief_accuracy/std': 0.011529605835676193, 'reward': 0.6347924470901489, 'reward_std': 0.019179200753569603, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.0008081197738647, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.45}
+
+
+ 28%|██▊       | 224/800 [1:19:10<2:34:53, 16.14s/it]
+
+{'loss': 0.0224, 'grad_norm': 3.4472429752349854, 'learning_rate': 4e-05, 'num_tokens': 4445293.0, 'completions/mean_length': 25.6875, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.65625, 'completions/mean_terminated_length': 13.636363983154297, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 27.0, 'rewards/format_valid/mean': -0.546875, 'rewards/format_valid/std': 1.3990744352340698, 'rewards/action_legal/mean': -0.46875, 'rewards/action_legal/std': 0.507007360458374, 'rewards/env_reward/mean': -0.9508461356163025, 'rewards/env_reward/std': 1.963952660560608, 'rewards/belief_accuracy/mean': -0.07500000298023224, 'rewards/belief_accuracy/std': 0.04399413242936134, 'reward': -1.7020504474639893, 'reward_std': 0.5680508613586426, 'frac_reward_zero_std': 0.5, 'completion_length': 32.0, 'kl': 0.5610938370227814, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.45}
+
+
+ 28%|██▊       | 225/800 [1:19:28<2:40:47, 16.78s/it]
+
+
+                                                     
+
+
+
+ 28%|██▊       | 225/800 [1:19:28<2:40:47, 16.78s/it]
+
+
+ 28%|██▊       | 226/800 [1:19:44<2:38:21, 16.55s/it]
+
+{'loss': 0.047, 'grad_norm': 5.46085262298584, 'learning_rate': 3.993055555555556e-05, 'num_tokens': 4465084.0, 'completions/mean_length': 18.46875, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.28125, 'completions/mean_terminated_length': 13.17391300201416, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 26.0, 'rewards/format_valid/mean': 0.4375, 'rewards/format_valid/std': 0.45348116755485535, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.5873256921768188, 'rewards/env_reward/std': 0.9479782581329346, 'rewards/belief_accuracy/mean': -0.0984220802783966, 'rewards/belief_accuracy/std': 0.008926110342144966, 'reward': 0.6060348153114319, 'reward_std': 0.6351835131645203, 'frac_reward_zero_std': 0.75, 'completion_length': 32.0, 'kl': 1.1745562702417374, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.45}
+
+
+                                                     
+
+
+
+ 28%|██▊       | 226/800 [1:19:44<2:38:21, 16.55s/it]
+
+{'loss': 0.04, 'grad_norm': 11.4356107711792, 'learning_rate': 3.986111111111111e-05, 'num_tokens': 4484947.0, 'completions/mean_length': 20.71875, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.28125, 'completions/mean_terminated_length': 16.30434799194336, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': 0.53125, 'rewards/format_valid/std': 0.12296734005212784, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.49249911308288574, 'rewards/env_reward/std': 0.2406158745288849, 'rewards/belief_accuracy/mean': -0.09860005974769592, 'rewards/belief_accuracy/std': 0.015982097014784813, 'reward': 0.4695109724998474, 'reward_std': 0.018066100776195526, 'frac_reward_zero_std': 0.5, 'completion_length': 32.0, 'kl': 1.0011040717363358, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.45}
+
+
+ 28%|██▊       | 227/800 [1:20:00<2:36:26, 16.38s/it]
+
+
+                                                     
+
+
+
+ 28%|██▊       | 227/800 [1:20:00<2:36:26, 16.38s/it]
+
+{'loss': 0.0447, 'grad_norm': 19.791118621826172, 'learning_rate': 3.979166666666667e-05, 'num_tokens': 4504725.0, 'completions/mean_length': 18.0625, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.28125, 'completions/mean_terminated_length': 12.608695983886719, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.53125, 'rewards/format_valid/std': 0.12296734005212784, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.27436113357543945, 'rewards/env_reward/std': 0.09100066870450974, 'rewards/belief_accuracy/mean': -0.09800823777914047, 'rewards/belief_accuracy/std': 0.0262263435870409, 'reward': 0.1440795212984085, 'reward_std': 0.06190364062786102, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.1169381141662598, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.46}
+
+
+ 28%|██▊       | 228/800 [1:20:16<2:35:05, 16.27s/it]
+
+
+                                                     
+
+
+
+ 28%|██▊       | 228/800 [1:20:16<2:35:05, 16.27s/it]
+
+{'loss': 0.0218, 'grad_norm': 3.973587989807129, 'learning_rate': 3.972222222222222e-05, 'num_tokens': 4524732.0, 'completions/mean_length': 25.21875, 'completions/min_length': 9.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.53125, 'completions/mean_terminated_length': 17.533334732055664, 'completions/min_terminated_length': 9.0, 'completions/max_terminated_length': 25.0, 'rewards/format_valid/mean': 0.28125, 'rewards/format_valid/std': 1.1354798078536987, 'rewards/action_legal/mean': -0.1875, 'rewards/action_legal/std': 0.3965577781200409, 'rewards/env_reward/mean': -0.11469113826751709, 'rewards/env_reward/std': 1.4347795248031616, 'rewards/belief_accuracy/mean': -0.07732638716697693, 'rewards/belief_accuracy/std': 0.07167626917362213, 'reward': -0.39932841062545776, 'reward_std': 0.7877496480941772, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 0.5447909384965897, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.46}
+
+
+ 29%|██▊       | 229/800 [1:20:32<2:34:02, 16.19s/it]
+
+
+                                                     
+
+
+
+ 29%|██▊       | 229/800 [1:20:32<2:34:02, 16.19s/it]
+
+
+ 29%|██▉       | 230/800 [1:20:48<2:33:13, 16.13s/it]
+
+
+                                                     
+
+
+{'loss': 0.0385, 'grad_norm': 1.3134355545043945, 'learning_rate': 3.9652777777777776e-05, 'num_tokens': 4544530.0, 'completions/mean_length': 18.6875, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.25, 'completions/mean_terminated_length': 14.25, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 31.0, 'rewards/format_valid/mean': 0.40625, 'rewards/format_valid/std': 0.6530017256736755, 'rewards/action_legal/mean': -0.0625, 'rewards/action_legal/std': 0.24593468010425568, 'rewards/env_reward/mean': 0.6369321942329407, 'rewards/env_reward/std': 1.0500494241714478, 'rewards/belief_accuracy/mean': -0.08488041162490845, 'rewards/belief_accuracy/std': 0.05245941877365112, 'reward': 0.7179445624351501, 'reward_std': 1.0996191501617432, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 0.9614753276109695, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.46}
+
+
+ 29%|██▉       | 230/800 [1:20:48<2:33:13, 16.13s/it]
+
+
+ 29%|██▉       | 231/800 [1:21:04<2:32:59, 16.13s/it]
+
+
+                                                     
+
+
+{'loss': 0.0549, 'grad_norm': 6.522164821624756, 'learning_rate': 3.958333333333333e-05, 'num_tokens': 4564256.0, 'completions/mean_length': 16.4375, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.125, 'completions/mean_terminated_length': 14.214286804199219, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.484375, 'rewards/format_valid/std': 0.48334142565727234, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.8155850172042847, 'rewards/env_reward/std': 0.7909724116325378, 'rewards/belief_accuracy/mean': -0.10208147764205933, 'rewards/belief_accuracy/std': 0.012231566943228245, 'reward': 0.939789354801178, 'reward_std': 0.5011690855026245, 'frac_reward_zero_std': 0.5, 'completion_length': 32.0, 'kl': 1.3722899556159973, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.46}
+
+
+ 29%|██▉       | 231/800 [1:21:04<2:32:59, 16.13s/it]
+
+{'loss': 0.0366, 'grad_norm': 83.5497817993164, 'learning_rate': 3.951388888888889e-05, 'num_tokens': 4584052.0, 'completions/mean_length': 18.625, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.25, 'completions/mean_terminated_length': 14.166666984558105, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.46875, 'rewards/format_valid/std': 0.4741290807723999, 'rewards/action_legal/mean': -0.03125, 'rewards/action_legal/std': 0.1767766922712326, 'rewards/env_reward/mean': 0.6184450387954712, 'rewards/env_reward/std': 0.7810394763946533, 'rewards/belief_accuracy/mean': -0.10046297311782837, 'rewards/belief_accuracy/std': 0.005624816752970219, 'reward': 0.6481536030769348, 'reward_std': 0.43163561820983887, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 0.913843423128128, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.46}
+
+
+ 29%|██▉       | 232/800 [1:21:20<2:32:30, 16.11s/it]
+
+
+                                                     
+
+
+
+ 29%|██▉       | 232/800 [1:21:20<2:32:30, 16.11s/it]
+
+
+ 29%|██▉       | 233/800 [1:21:36<2:32:00, 16.08s/it]
+
+
+                                                     
+
+
+{'loss': 0.0416, 'grad_norm': 1.2989977598190308, 'learning_rate': 3.944444444444445e-05, 'num_tokens': 4603842.0, 'completions/mean_length': 18.4375, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.21875, 'completions/mean_terminated_length': 14.639999389648438, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 32.0, 'rewards/format_valid/mean': 0.421875, 'rewards/format_valid/std': 0.6612472534179688, 'rewards/action_legal/mean': -0.0625, 'rewards/action_legal/std': 0.24593468010425568, 'rewards/env_reward/mean': 0.20217597484588623, 'rewards/env_reward/std': 0.9103991985321045, 'rewards/belief_accuracy/mean': -0.09981748461723328, 'rewards/belief_accuracy/std': 0.026089416816830635, 'reward': 0.02178022265434265, 'reward_std': 0.843180239200592, 'frac_reward_zero_std': 0.5, 'completion_length': 32.0, 'kl': 1.0395325869321823, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.47}
+
+
+ 29%|██▉       | 233/800 [1:21:36<2:32:00, 16.08s/it]
+
+
+ 29%|██▉       | 234/800 [1:21:52<2:31:30, 16.06s/it]
+
+
+                                                     
+
+
+{'loss': 0.0358, 'grad_norm': 9.895654678344727, 'learning_rate': 3.9375e-05, 'num_tokens': 4623601.0, 'completions/mean_length': 17.46875, 'completions/min_length': 6.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.25, 'completions/mean_terminated_length': 12.625, 'completions/min_terminated_length': 6.0, 'completions/max_terminated_length': 25.0, 'rewards/format_valid/mean': 0.53125, 'rewards/format_valid/std': 0.12296734005212784, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.9071379899978638, 'rewards/env_reward/std': 0.394985556602478, 'rewards/belief_accuracy/mean': -0.09447917342185974, 'rewards/belief_accuracy/std': 0.02241046354174614, 'reward': 1.1038320064544678, 'reward_std': 0.06022049859166145, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 0.8951070010662079, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.47}
+
+
+ 29%|██▉       | 234/800 [1:21:52<2:31:30, 16.06s/it]
+
+
+ 29%|██▉       | 235/800 [1:22:08<2:31:02, 16.04s/it]
+
+
+                                                     
+
+
+{'loss': 0.0436, 'grad_norm': 2.068256139755249, 'learning_rate': 3.9305555555555556e-05, 'num_tokens': 4643374.0, 'completions/mean_length': 17.90625, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.1875, 'completions/mean_terminated_length': 14.653846740722656, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 29.0, 'rewards/format_valid/mean': 0.5625, 'rewards/format_valid/std': 0.16800537705421448, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.24769337475299835, 'rewards/env_reward/std': 0.6311713457107544, 'rewards/belief_accuracy/mean': -0.09339796006679535, 'rewards/belief_accuracy/std': 0.029752185568213463, 'reward': 0.11947113275527954, 'reward_std': 0.05775392800569534, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.0895096063613892, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.47}
+
+
+ 29%|██▉       | 235/800 [1:22:08<2:31:02, 16.04s/it]
+
+{'loss': 0.0362, 'grad_norm': 2.276850461959839, 'learning_rate': 3.923611111111111e-05, 'num_tokens': 4663156.0, 'completions/mean_length': 18.1875, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.1875, 'completions/mean_terminated_length': 15.000000953674316, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.6875, 'rewards/format_valid/std': 0.24593468010425568, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 1.0912227630615234, 'rewards/env_reward/std': 0.4555222690105438, 'rewards/belief_accuracy/mean': -0.086835116147995, 'rewards/belief_accuracy/std': 0.0682639628648758, 'reward': 1.4107036590576172, 'reward_std': 0.14337408542633057, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 0.9056383520364761, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.47}
+
+
+ 30%|██▉       | 236/800 [1:22:24<2:30:37, 16.02s/it]
+
+
+                                                     
+
+
+
+ 30%|██▉       | 236/800 [1:22:24<2:30:37, 16.02s/it]
+
+
+ 30%|██▉       | 237/800 [1:22:40<2:30:25, 16.03s/it]
+
+
+                                                     
+
+
+{'loss': 0.0485, 'grad_norm': 3.727612018585205, 'learning_rate': 3.9166666666666665e-05, 'num_tokens': 4682990.0, 'completions/mean_length': 19.8125, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.15625, 'completions/mean_terminated_length': 17.55555534362793, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 28.0, 'rewards/format_valid/mean': 0.6875, 'rewards/format_valid/std': 0.24593468010425568, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.5215833187103271, 'rewards/env_reward/std': 0.5706321001052856, 'rewards/belief_accuracy/mean': -0.10854831337928772, 'rewards/belief_accuracy/std': 0.02328098565340042, 'reward': 0.49110499024391174, 'reward_std': 0.047724559903144836, 'frac_reward_zero_std': 0.25, 'completion_length': 32.0, 'kl': 1.2118152678012848, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.47}
+
+
+ 30%|██▉       | 237/800 [1:22:40<2:30:25, 16.03s/it]
+
+
+ 30%|██▉       | 238/800 [1:22:56<2:30:04, 16.02s/it]
+
+
+                                                     
+
+
+{'loss': 0.042, 'grad_norm': 1.0679957866668701, 'learning_rate': 3.909722222222223e-05, 'num_tokens': 4702830.0, 'completions/mean_length': 20.0, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.25, 'completions/mean_terminated_length': 16.0, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 29.0, 'rewards/format_valid/mean': 0.546875, 'rewards/format_valid/std': 0.1480722874403, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.3043600022792816, 'rewards/env_reward/std': 0.6461636424064636, 'rewards/belief_accuracy/mean': -0.10364583879709244, 'rewards/belief_accuracy/std': 0.012511196546256542, 'reward': 0.17294615507125854, 'reward_std': 0.022001968696713448, 'frac_reward_zero_std': 0.5, 'completion_length': 32.0, 'kl': 1.0511298477649689, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.48}
+
+
+ 30%|██▉       | 238/800 [1:22:56<2:30:04, 16.02s/it]
+
+
+ 30%|██▉       | 239/800 [1:23:12<2:29:51, 16.03s/it]
+
+
+                                                     
+
+
+{'loss': 0.0437, 'grad_norm': 33.40415573120117, 'learning_rate': 3.902777777777778e-05, 'num_tokens': 4722691.0, 'completions/mean_length': 20.65625, 'completions/min_length': 7.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.3125, 'completions/mean_terminated_length': 15.5, 'completions/min_terminated_length': 7.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.609375, 'rewards/format_valid/std': 0.2100067138671875, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.9613187313079834, 'rewards/env_reward/std': 0.7064112424850464, 'rewards/belief_accuracy/mean': -0.07955603301525116, 'rewards/belief_accuracy/std': 0.05856386944651604, 'reward': 1.233778715133667, 'reward_std': 0.18115174770355225, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 1.0937408059835434, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.48}
+
+
+ 30%|██▉       | 239/800 [1:23:12<2:29:51, 16.03s/it]
+
+{'loss': 0.0404, 'grad_norm': 16.607942581176758, 'learning_rate': 3.8958333333333336e-05, 'num_tokens': 4742532.0, 'completions/mean_length': 20.03125, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.28125, 'completions/mean_terminated_length': 15.34782600402832, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 26.0, 'rewards/format_valid/mean': 0.515625, 'rewards/format_valid/std': 0.0883883461356163, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.7018182277679443, 'rewards/env_reward/std': 0.49033573269844055, 'rewards/belief_accuracy/mean': -0.10125000774860382, 'rewards/belief_accuracy/std': 0.007071067579090595, 'reward': 0.774758517742157, 'reward_std': 0.04344534873962402, 'frac_reward_zero_std': 0.5, 'completion_length': 32.0, 'kl': 1.0090321898460388, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.48}
+
+
+ 30%|███       | 240/800 [1:23:28<2:29:23, 16.01s/it]
+
+
+                                                     
+
+
+
+ 30%|███       | 240/800 [1:23:28<2:29:23, 16.01s/it]
+
+
+ 30%|███       | 241/800 [1:23:44<2:28:59, 15.99s/it]
+
+
+                                                     
+
+
+{'loss': 0.0361, 'grad_norm': 9.747608184814453, 'learning_rate': 3.888888888888889e-05, 'num_tokens': 4762383.0, 'completions/mean_length': 20.34375, 'completions/min_length': 8.0, 'completions/max_length': 32.0, 'completions/clipped_ratio': 0.3125, 'completions/mean_terminated_length': 15.045454978942871, 'completions/min_terminated_length': 8.0, 'completions/max_terminated_length': 30.0, 'rewards/format_valid/mean': 0.6875, 'rewards/format_valid/std': 0.24593468010425568, 'rewards/action_legal/mean': 0.0, 'rewards/action_legal/std': 0.0, 'rewards/env_reward/mean': 0.8115883469581604, 'rewards/env_reward/std': 0.39453771710395813, 'rewards/belief_accuracy/mean': -0.10486429184675217, 'rewards/belief_accuracy/std': 0.0323609933257103, 'reward': 0.9371646642684937, 'reward_std': 0.1926143914461136, 'frac_reward_zero_std': 0.0, 'completion_length': 32.0, 'kl': 0.9028782099485397, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'epoch': 0.48}
+
+
+ 30%|███       | 241/800 [1:23:44<2:28:59, 15.99s/it]
\ No newline at end of file