Upload folder using huggingface_hub

train/hour1_smoke.py

@@ -42,13 +42,15 @@ def main():
         traceback.print_exc()
         sys.exit(1)
 
-    # 2. Unsloth + TRL imports
-    print("\n[2/6] importing Unsloth + TRL")
+    # 2. transformers + TRL + PEFT (deck requirement #2: "Unsloth OR HF TRL")
+    print("\n[2/6] importing transformers + TRL + PEFT")
     try:
-        from unsloth import FastLanguageModel
+        import torch as _t
+        from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+        from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
         from trl import GRPOTrainer, GRPOConfig
         from datasets import Dataset
-        print("   ✓ Unsloth + TRL + datasets imported")
+        print("   ✓ transformers + TRL + PEFT + datasets imported")
     except Exception as e:
         print(f"   ✗ {e}")
         traceback.print_exc()
@@ -80,23 +82,31 @@ def main():
         traceback.print_exc()
         sys.exit(1)
 
-    # 5. Load Qwen2.5-3B-Instruct + LoRA
-    print("\n[5/6] loading Qwen2.5-3B-Instruct (4-bit + LoRA)")
+    # 5. Load Qwen2.5-3B-Instruct + LoRA via plain transformers + PEFT
+    print("\n[5/6] loading Qwen2.5-3B-Instruct (4-bit + LoRA via transformers/peft)")
     try:
-        import torch as _t
-        model, tokenizer = FastLanguageModel.from_pretrained(
-            model_name="unsloth/Qwen2.5-3B-Instruct",
-            max_seq_length=2048,  # smaller than full 4096 for speed
+        bnb = BitsAndBytesConfig(
             load_in_4bit=True,
-            dtype=_t.bfloat16,    # avoid LoRA dtype mismatch on L4
+            bnb_4bit_compute_dtype=_t.bfloat16,
+            bnb_4bit_quant_type="nf4",
+            bnb_4bit_use_double_quant=True,
+        )
+        tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-3B-Instruct")
+        if tokenizer.pad_token is None:
+            tokenizer.pad_token = tokenizer.eos_token
+        base = AutoModelForCausalLM.from_pretrained(
+            "Qwen/Qwen2.5-3B-Instruct",
+            quantization_config=bnb,
+            dtype=_t.bfloat16,
+            device_map="auto",
         )
-        model = FastLanguageModel.get_peft_model(
-            model,
-            r=8,                   # smaller r for the smoke test
-            lora_alpha=16,
+        base = prepare_model_for_kbit_training(base, use_gradient_checkpointing=True)
+        peft_config = LoraConfig(
+            r=8, lora_alpha=16, lora_dropout=0.0, bias="none",
             target_modules=["q_proj", "k_proj", "v_proj", "o_proj"],
-            use_gradient_checkpointing=True,   # plain torch GC, not "unsloth" custom
+            task_type="CAUSAL_LM",
         )
+        model = get_peft_model(base, peft_config)
         n_trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
         print(f"   ✓ model loaded; {n_trainable / 1e6:.1f}M LoRA params trainable")
     except Exception as e:

train/train_grpo.py

@@ -255,7 +255,7 @@ def reward_decomposition(text: str, gold: str) -> Dict[str, float]:
 
 def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument("--model", default="unsloth/Qwen2.5-3B-Instruct")
+    parser.add_argument("--model", default="Qwen/Qwen2.5-3B-Instruct")
     parser.add_argument("--output-dir", default="./checkpoints/run1")
     parser.add_argument("--max-steps", type=int, default=200)
     parser.add_argument("--num-generations", type=int, default=4)
@@ -278,25 +278,41 @@ def main():
     dataset = build_dataset(scenarios, n_rows=args.n_train_rows)
     print(f"[data] {len(dataset)} prompt rows built")
 
-    # Model load
-    from unsloth import FastLanguageModel
+    # Model load via plain transformers + PEFT (deck-compliant: training uses HF TRL).
+    # We dropped Unsloth because their fast_lora kernel has a Half/Float dtype
+    # mismatch on Qwen2.5-3B + 4-bit + bf16 in v2026.4.8 (verified via failed
+    # smoke runs on L4). Plain transformers+peft+trl is slower but reliable.
+    import torch as _t
+    from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+    from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
     from trl import GRPOTrainer, GRPOConfig
 
     print(f"[load] {args.model}, 4-bit, max_seq_length={args.seq_length}")
-    import torch as _t
-    model, tokenizer = FastLanguageModel.from_pretrained(
-        model_name=args.model,
-        max_seq_length=args.seq_length,
+    bnb = BitsAndBytesConfig(
         load_in_4bit=True,
-        dtype=_t.bfloat16,    # explicit dtype prevents LoRA Half/Float mismatch
+        bnb_4bit_compute_dtype=_t.bfloat16,
+        bnb_4bit_quant_type="nf4",
+        bnb_4bit_use_double_quant=True,
+    )
+    # Strip the "unsloth/" prefix if the user passed an Unsloth-prefixed name —
+    # we now load directly from the upstream Qwen repo.
+    model_name = args.model.replace("unsloth/", "Qwen/").replace("-Instruct-bnb-4bit", "-Instruct")
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
+    base = AutoModelForCausalLM.from_pretrained(
+        model_name,
+        quantization_config=bnb,
+        dtype=_t.bfloat16,
+        device_map="auto",
     )
-    model = FastLanguageModel.get_peft_model(
-        model,
-        r=args.lora_r,
-        lora_alpha=args.lora_r,
+    base = prepare_model_for_kbit_training(base, use_gradient_checkpointing=True)
+    peft_config = LoraConfig(
+        r=args.lora_r, lora_alpha=args.lora_r, lora_dropout=0.0, bias="none",
         target_modules=["q_proj", "k_proj", "v_proj", "o_proj"],
-        use_gradient_checkpointing=True,   # plain torch GC; avoids unsloth-zoo dtype bug
+        task_type="CAUSAL_LM",
     )
+    model = get_peft_model(base, peft_config)
 
     config = GRPOConfig(
         output_dir=args.output_dir,