Rohan03 commited on
Commit
a1382fa
·
verified ·
1 Parent(s): 3177d8a

Fix Issue 4: Make mock heuristic detection more resilient

Browse files
Files changed (1) hide show
  1. patches/validate_mock_fix.patch +11 -0
patches/validate_mock_fix.patch ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ --- a/benchmarks/validate.py
2
+ +++ b/benchmarks/validate.py
3
+ @@ -107,7 +107,7 @@
4
+ def make_mock(task_name):
5
+ mock = MockLLMBackend()
6
+ t = TASKS[task_name]
7
+ def actor(msgs):
8
+ text = " ".join(m.content for m in msgs)
9
+ - has_h = "Learned Strategies" in text and "None yet" not in text
10
+ + has_h = ("Learned Strategies" in text or "When:" in text) and "None yet" not in text
11
+ code = t["good"] if has_h else t["bad"]