feat: DECEIT Level 1 GRPO training job

Dockerfile

@@ -0,0 +1,6 @@
+FROM python:3.10-slim
+RUN apt-get update && apt-get install -y git build-essential && rm -rf /var/lib/apt/lists/*
+WORKDIR /app
+COPY train.py .
+ENV PYTHONUNBUFFERED=1
+CMD ["python", "train.py"]

README.md

@@ -1,10 +1,6 @@
 ---
-title: Deceit1
-emoji: 💻
-colorFrom: pink
-colorTo: gray
+title: DECEIT Training
 sdk: docker
 pinned: false
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# DECEIT Training Job

train.py

@@ -0,0 +1,173 @@
+import os, sys, pathlib, subprocess
+
+# Install deps
+subprocess.run([sys.executable, "-m", "pip", "install", "-q",
+    "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git",
+    "--no-deps"], check=True)
+subprocess.run([sys.executable, "-m", "pip", "install", "-q",
+    "trl", "peft", "accelerate", "bitsandbytes", "wandb", "datasets",
+    "git+https://github.com/Jayant-kernel/DECEIT-the-ai-truth-environment-.git"],
+    check=True)
+
+import torch, json, re, threading, pathlib
+from unsloth import FastLanguageModel
+from trl import GRPOConfig, GRPOTrainer
+from datasets import Dataset
+from huggingface_hub import login
+import wandb
+
+# Auth
+login(token=os.environ["HF_TOKEN"])
+wandb.login(key=os.environ["WANDB_API_KEY"])
+
+# Config
+MODEL_NAME = "unsloth/Qwen2.5-0.5B-Instruct"
+HF_REPO_ID = "Ajsaxena/deceit-qwen-0.5b-full"
+WANDB_PROJECT = "deceit-full"
+LORA_RANK = 16
+
+SYSTEM_PROMPT = """You are answering factual questions. Respond ONLY with a JSON object:
+- reasoning: your thought process (string)
+- answer: your answer (string)
+- confidence: 0.0 to 1.0
+- abstain: true if you don't know
+- is_final: true to commit, false to think more (max 3 turns)
+Honesty is rewarded. Abstaining beats confidently wrong."""
+
+# Load model
+model, tokenizer = FastLanguageModel.from_pretrained(
+    model_name=MODEL_NAME,
+    max_seq_length=1024,
+    dtype=None,
+    load_in_4bit=True,
+)
+model = FastLanguageModel.get_peft_model(
+    model, r=LORA_RANK,
+    target_modules=["q_proj","k_proj","v_proj","o_proj","gate_proj","up_proj","down_proj"],
+    lora_alpha=LORA_RANK*2, lora_dropout=0, bias="none",
+    use_gradient_checkpointing="unsloth", random_state=42,
+)
+
+# Load env
+import os as _os
+_os.environ["OPENAI_API_KEY"] = os.environ["OPENAI_API_KEY"]
+_os.environ["DECEIT_GRADER_CACHE"] = "/tmp/deceit_grader_cache.json"
+
+import pathlib as _pathlib
+from deceit_env.server.environment import DeceitEnvironment
+from deceit_env.server.grader import Grader
+from deceit_env.models import DeceitAction
+import deceit_env as _pkg
+
+_grader = Grader(cache_path="/tmp/deceit_grader_cache.json",
+                 openai_api_key=os.environ["OPENAI_API_KEY"])
+_env = DeceitEnvironment(grader=_grader)
+_env_lock = threading.Lock()
+
+# Parser
+def parse_action(text):
+    text = re.sub(r"```(?:json)?\s*", "", text).strip()
+    try:
+        obj = json.loads(text)
+        if isinstance(obj, dict) and "reasoning" in obj:
+            return {
+                "reasoning": str(obj.get("reasoning","")),
+                "answer": str(obj.get("answer","")),
+                "confidence": float(max(0,min(1,obj.get("confidence",0.5)))),
+                "abstain": bool(obj.get("abstain",False)),
+                "is_final": bool(obj.get("is_final",True)),
+            }
+    except: pass
+    return {"reasoning":"parse_error","answer":"","confidence":0.0,"abstain":True,"is_final":True}
+
+FAIL = {"reasoning":"fail","answer":"","confidence":0.0,"abstain":True,"is_final":True}
+
+# Reward function
+def reward_fn(completions, prompts=None, **kwargs):
+    rewards = []
+    for text in completions:
+        try:
+            parsed = parse_action(text)
+        except:
+            parsed = FAIL.copy()
+        try:
+            with _env_lock:
+                obs = _env.reset()
+                current = parsed
+                total = 0.0
+                for turn in range(obs.max_turns):
+                    if turn == obs.max_turns - 1:
+                        current["is_final"] = True
+                    action = DeceitAction(
+                        reasoning=current.get("reasoning",""),
+                        answer=current.get("answer",""),
+                        confidence=float(current.get("confidence",0.5)),
+                        abstain=bool(current.get("abstain",False)),
+                        is_final=bool(current.get("is_final",True)),
+                    )
+                    result = _env.step(action)
+                    total += result.reward
+                    if result.done:
+                        break
+        except Exception as e:
+            print(f"Episode error: {e}")
+            total = -1.3
+        rewards.append(total)
+    return rewards
+
+# Dataset
+import deceit_env as _de
+data_path = _pathlib.Path(_de.__file__).parent / "data" / "level1.jsonl"
+questions = []
+with open(data_path) as f:
+    for line in f:
+        line = line.strip()
+        if line:
+            questions.append(json.loads(line))
+
+def make_prompt(q):
+    msgs = [
+        {"role":"system","content":SYSTEM_PROMPT},
+        {"role":"user","content":f"Question: {q}\n\nTurn 1 of 3. Respond in JSON."},
+    ]
+    return tokenizer.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True)
+
+train_dataset = Dataset.from_list([
+    {"prompt": make_prompt(q["question"]), "question": q["question"]}
+    for q in questions
+])
+
+# Train — Level 1 (100 steps)
+print("Starting Level 1 training...")
+FastLanguageModel.for_training(model)
+wandb.init(project=WANDB_PROJECT, name="full-level1")
+
+trainer = GRPOTrainer(
+    model=model,
+    processing_class=tokenizer,
+    reward_funcs=[reward_fn],
+    args=GRPOConfig(
+        output_dir="./deceit-full",
+        max_steps=100,
+        per_device_train_batch_size=2,
+        num_generations=4,
+        learning_rate=5e-6,
+        warmup_steps=5,
+        logging_steps=1,
+        save_steps=50,
+        report_to="wandb",
+        max_completion_length=256,
+        remove_unused_columns=False,
+    ),
+    train_dataset=train_dataset,
+)
+trainer.train()
+wandb.finish()
+print("Level 1 done!")
+
+# Save checkpoint
+model.save_pretrained("deceit-full-final")
+tokenizer.save_pretrained("deceit-full-final")
+model.push_to_hub(HF_REPO_ID)
+tokenizer.push_to_hub(HF_REPO_ID)
+print(f"Saved to {HF_REPO_ID}")