fix: debug model output parsing in evaluation

evaluate.py

@@ -61,9 +61,9 @@ SYSTEM_PROMPT = """You are answering factual questions. Respond ONLY with a JSON
 import re
 
 def parse_action(text):
-    text = re.sub(r"```(?:json)?\s*", "", text).strip()
+    cleaned = re.sub(r"```(?:json)?\s*", "", text).strip()
     try:
-        obj = json.loads(text)
+        obj = json.loads(cleaned)
         if isinstance(obj, dict) and "reasoning" in obj:
             return {
                 "reasoning": str(obj.get("reasoning", "")),
@@ -74,6 +74,15 @@ def parse_action(text):
             }
     except:
         pass
+    # Try to extract answer from plain text patterns
+    answer = ""
+    m = re.search(r'"answer"\s*:\s*"([^"]+)"', cleaned)
+    if m:
+        answer = m.group(1)
+    elif re.search(r'\b(yes|no|true|false)\b', cleaned, re.I):
+        answer = re.search(r'\b(yes|no|true|false)\b', cleaned, re.I).group(0)
+    if answer:
+        return {"reasoning": cleaned[:100], "answer": answer, "confidence": 0.4, "abstain": False, "is_final": True}
     return {"reasoning":"","answer":"","confidence":0.0,"abstain":True,"is_final":True}
 
 def evaluate_model(model_name, label, n_episodes=30, is_trained=False):
@@ -140,7 +149,9 @@ def evaluate_model(model_name, label, n_episodes=30, is_trained=False):
             skip_special_tokens=True
         )
 
+        print(f"Model output: {text[:200]}")
         parsed = parse_action(text)
+        print(f"Parsed: {parsed}")
 
         if parsed["abstain"]:
             results["abstain"] += 1