add hub push on every save + save every 4000 steps for crash recovery

train_hrm_text_pi.py

@@ -929,19 +929,21 @@ def main():
         warmup_steps=500 if not args.test else 0,
         lr_scheduler_type="cosine",
         eval_strategy="epoch",
-        save_strategy="epoch",
+        save_strategy="steps",
+        save_steps=4000,
         load_best_model_at_end=True,
         metric_for_best_model="f1",
         greater_is_better=True,
-        save_total_limit=2,
+        save_total_limit=3,
         logging_strategy="steps",
         logging_first_step=True,
         logging_steps=5 if args.test else 20,
         disable_tqdm=False if args.test else True,
         fp16=use_cuda,
         bf16=False,
-        push_to_hub=False,
-        hub_model_id=None,
+        push_to_hub=True,
+        hub_model_id=args.push_to_hub,
+        hub_strategy="every_save",
         use_cpu=not use_cuda,
         dataloader_num_workers=4,
         seed=args.seed,