Training in progress, step 25395, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b64a8c5c9c45c847ee9749e308f164cca0b62ea7a0eadbd62743209b62d4936c
+oid sha256:496ff499aedb0ce97ed3a66cc050ade4c7b18f28ddfc368d0252dcde6c6caaff
 size 1230207488

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9625bd0c592b0f12935bb339b14cb4f020840a853ccee3f8bb7b7e44eb6fe2fb
+oid sha256:8a226c815c64a6223d2e29d0b7924537ea47ca35d20305489fac23083648d163
 size 2460722266

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:977dcbce44dc112e81ce232ba292a2987f23c3109ccace9ad089d6fe42b17db9
+oid sha256:7a11e2e02a23ac5bb22f239d204a60875ed400da34d9862f2092a8bbfa558025
 size 14645

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3c9afd38371c0ea8a30afeea2a83b5feda14966e61c68d2e39fa59a47c408813
+oid sha256:8584a03231a936447aeb179c66bfa2afda8aa94439b68eb56489d3fd90d88397
 size 1465

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": 23000,
   "best_metric": 1.7628060579299927,
   "best_model_checkpoint": "/home/ubuntu/sangoai-training/output/nllb-sango-finetuned/checkpoint-23000",
-  "epoch": 2.9534191843792623,
+  "epoch": 3.0,
   "eval_steps": 500,
-  "global_step": 25000,
+  "global_step": 25395,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -3908,6 +3908,63 @@
       "eval_samples_per_second": 484.683,
       "eval_steps_per_second": 40.396,
       "step": 25000
+    },
+    {
+      "epoch": 2.9593263664386136,
+      "grad_norm": 8.0,
+      "learning_rate": 6.949186583651336e-07,
+      "loss": 20.5435107421875,
+      "step": 25050
+    },
+    {
+      "epoch": 2.965233548497965,
+      "grad_norm": 8.5,
+      "learning_rate": 5.944968869250854e-07,
+      "loss": 20.75857666015625,
+      "step": 25100
+    },
+    {
+      "epoch": 2.9711407305573156,
+      "grad_norm": 7.96875,
+      "learning_rate": 4.940751154850371e-07,
+      "loss": 20.8127978515625,
+      "step": 25150
+    },
+    {
+      "epoch": 2.977047912616667,
+      "grad_norm": 8.8125,
+      "learning_rate": 3.936533440449889e-07,
+      "loss": 20.79349853515625,
+      "step": 25200
+    },
+    {
+      "epoch": 2.9829550946760177,
+      "grad_norm": 8.3125,
+      "learning_rate": 2.932315726049408e-07,
+      "loss": 20.7541015625,
+      "step": 25250
+    },
+    {
+      "epoch": 2.988862276735369,
+      "grad_norm": 8.9375,
+      "learning_rate": 1.9280980116489258e-07,
+      "loss": 20.79868408203125,
+      "step": 25300
+    },
+    {
+      "epoch": 2.9947694587947202,
+      "grad_norm": 8.1875,
+      "learning_rate": 9.238802972484435e-08,
+      "loss": 20.6675341796875,
+      "step": 25350
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 1.7628397941589355,
+      "eval_runtime": 121.1922,
+      "eval_samples_per_second": 485.205,
+      "eval_steps_per_second": 40.44,
+      "step": 25395
     }
   ],
   "logging_steps": 50,
@@ -3922,12 +3979,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 2.6144185253894554e+17,
+  "total_flos": 2.655110146920284e+17,
   "train_batch_size": 12,
   "trial_name": null,
   "trial_params": null