Training completed

README.md

@@ -38,8 +38,10 @@ The following hyperparameters were used during training:
 - eval_batch_size: 8
 - seed: 42
 - distributed_type: multi-GPU
+- num_devices: 2
 - gradient_accumulation_steps: 10
-- total_train_batch_size: 20
+- total_train_batch_size: 40
+- total_eval_batch_size: 16
 - optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.05
@@ -55,4 +57,4 @@ The following hyperparameters were used during training:
 - Transformers 4.51.3
 - Pytorch 2.3.0+cu121
 - Datasets 2.15.0
-- Tokenizers 0.21.2
+- Tokenizers 0.21.4

adapter_config.json

@@ -24,12 +24,12 @@
   "revision": null,
   "target_modules": [
     "v_proj",
+    "q_proj",
+    "down_proj",
+    "o_proj",
     "k_proj",
     "up_proj",
-    "gate_proj",
-    "down_proj",
-    "q_proj",
-    "o_proj"
+    "gate_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:40179f18b5d2cbc4f5334b8b416bea9dba2886ec9136b756b5516f5eddb7ccad
+oid sha256:d035c7d217578a4cf81b6f3d9058e6b661d834dc982c560bb61d99f7eb41ab0b
 size 11301520

all_results.json

@@ -1,8 +1,11 @@
 {
-    "epoch": 0.005,
-    "total_flos": 2405455221489664.0,
-    "train_loss": 3.154673767089844,
-    "train_runtime": 69.8938,
-    "train_samples_per_second": 2.861,
-    "train_steps_per_second": 0.143
+    "avg_step_time": 6.669609618186951,
+    "epoch": 0.01,
+    "total_flos": 4810910442979328.0,
+    "total_training_time": 89.90108489990234,
+    "total_training_time_mins": 1.4983514149983723,
+    "train_loss": 3.1294017791748048,
+    "train_runtime": 82.5533,
+    "train_samples_per_second": 4.845,
+    "train_steps_per_second": 0.121
 }

tokenizer_config.json

@@ -2050,6 +2050,7 @@
     }
   },
   "bos_token": "<|begin_of_text|>",
+  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{'<|im_start|>assistant\n'}}{% endif %}",
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|end_of_text|>",
   "extra_special_tokens": {},

train_results.json

@@ -1,8 +1,11 @@
 {
-    "epoch": 0.005,
-    "total_flos": 2405455221489664.0,
-    "train_loss": 3.154673767089844,
-    "train_runtime": 69.8938,
-    "train_samples_per_second": 2.861,
-    "train_steps_per_second": 0.143
+    "avg_step_time": 6.669609618186951,
+    "epoch": 0.01,
+    "total_flos": 4810910442979328.0,
+    "total_training_time": 89.90108489990234,
+    "total_training_time_mins": 1.4983514149983723,
+    "train_loss": 3.1294017791748048,
+    "train_runtime": 82.5533,
+    "train_samples_per_second": 4.845,
+    "train_steps_per_second": 0.121
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.005,
+  "epoch": 0.01,
   "eval_steps": 0,
   "global_step": 10,
   "is_hyper_param_search": false,
@@ -10,20 +10,20 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.005,
-      "grad_norm": 1.493022084236145,
+      "epoch": 0.01,
+      "grad_norm": 0.5258329510688782,
       "learning_rate": 0.0001,
-      "loss": 3.1547,
+      "loss": 3.1294,
       "step": 10
     },
     {
-      "epoch": 0.005,
+      "epoch": 0.01,
       "step": 10,
-      "total_flos": 2405455221489664.0,
-      "train_loss": 3.154673767089844,
-      "train_runtime": 69.8938,
-      "train_samples_per_second": 2.861,
-      "train_steps_per_second": 0.143
+      "total_flos": 4810910442979328.0,
+      "train_loss": 3.1294017791748048,
+      "train_runtime": 82.5533,
+      "train_samples_per_second": 4.845,
+      "train_steps_per_second": 0.121
     }
   ],
   "logging_steps": 10,
@@ -43,7 +43,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2405455221489664.0,
+  "total_flos": 4810910442979328.0,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b5b476429fe4dd0fdd2be265d8440ed606c5801b51bea1e75391a43581bf7dd3
-size 9528
+oid sha256:6302d7204c11d65102ccfe9a727ca8d3bba190dd19170c0e413b309c5b094043
+size 8440