Fix Issue 4: Make mock heuristic detection more resilient
Browse files
patches/validate_mock_fix.patch
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
--- a/benchmarks/validate.py
|
| 2 |
+
+++ b/benchmarks/validate.py
|
| 3 |
+
@@ -107,7 +107,7 @@
|
| 4 |
+
def make_mock(task_name):
|
| 5 |
+
mock = MockLLMBackend()
|
| 6 |
+
t = TASKS[task_name]
|
| 7 |
+
def actor(msgs):
|
| 8 |
+
text = " ".join(m.content for m in msgs)
|
| 9 |
+
- has_h = "Learned Strategies" in text and "None yet" not in text
|
| 10 |
+
+ has_h = ("Learned Strategies" in text or "When:" in text) and "None yet" not in text
|
| 11 |
+
code = t["good"] if has_h else t["bad"]
|