Upload folder using huggingface_hub

Dockerfile

@@ -13,11 +13,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     curl \
     && rm -rf /var/lib/apt/lists/*
 
-COPY requirements.txt ./
+COPY requirements.txt /app/server/requirements.txt
 RUN pip install --no-cache-dir --upgrade pip && \
-    pip install --no-cache-dir -r requirements.txt
+    pip install --no-cache-dir -r /app/server/requirements.txt
 
-COPY . .
+COPY . /app/server
+
+WORKDIR /app/server
 
 EXPOSE 8000
 

inference.py

@@ -275,9 +275,7 @@ def select_first_action(task_id: str, llm_action: dict[str, Any]) -> dict[str, A
     """Prefer a safe model suggestion, otherwise use the deterministic fallback."""
     action_type = safe_text(llm_action.get("action_type", ""), "")
     code = llm_action.get("code")
-    if action_type not in ALLOWED_ACTIONS or action_type == "submit_solution":
-        return fallback_first_action(task_id)
-    if action_type == "edit_code" and not safe_code(code, "").strip():
+    if action_type not in {"analyze_code", "run_tests"}:
         return fallback_first_action(task_id)
     return {"action_type": action_type, "code": code}
 
@@ -323,10 +321,14 @@ def run_task(task_id: str, client: Any | None, model: str) -> None:
     step_count = 0
     llm_action = run_llm(client, model, build_prompt(observation))
     reference_code = safe_reference_code(task_id, safe_code(safe_getattr(observation, "current_code", ""), ""))
+    first_action = select_first_action(task_id, llm_action)
+    second_action = {"action_type": "run_tests", "code": None} if first_action["action_type"] == "analyze_code" else {"action_type": "analyze_code", "code": None}
     planned_actions = [
-        select_first_action(task_id, llm_action),
+        first_action,
+        second_action,
+        {"action_type": "analyze_code", "code": None},
+        {"action_type": "run_tests", "code": None},
         {"action_type": "edit_code", "code": reference_code},
-        {"action_type": "submit_solution", "code": None},
     ]
 
     final_observation = observation

requirements.txt

@@ -1,5 +1,5 @@
 fastapi>=0.115,<1.0
-uvicorn[standard]>=0.30,<1.0
+uvicorn>=0.30,<1.0
 pydantic>=2.0,<3.0
 openai>=1.0,<3.0
 pytest>=8.0,<9.0

server/Dockerfile

@@ -7,21 +7,23 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
     WORKERS=1 \
     MAX_CONCURRENT_ENVS=16
 
-WORKDIR /app
+WORKDIR /app
 
 # Install system dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
     curl \
     && rm -rf /var/lib/apt/lists/*
 
-# Install Python dependencies
-COPY requirements.txt ./
-RUN pip install --no-cache-dir --upgrade pip && \
-    pip install --no-cache-dir -r requirements.txt
-
-# Copy the self-contained server package
-COPY . /app/server
-
-# Run FastAPI app
-EXPOSE ${PORT}
-CMD ["python", "-m", "server.app"]
+# Install Python dependencies
+COPY requirements.txt /app/server/requirements.txt
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r /app/server/requirements.txt
+
+# Copy the self-contained server package
+COPY . /app/server
+
+WORKDIR /app/server
+
+# Run FastAPI app
+EXPOSE ${PORT}
+CMD ["python", "-m", "server.app"]

server/env_safe.py

@@ -34,12 +34,8 @@ except Exception:
 
 
 INVALID_ACTION_PENALTY = 0.10
-NO_PROGRESS_PENALTY = 0.08
-REPEATED_ACTION_PENALTY = 0.05
-BASE_STEP_PENALTY = 0.02
-ANALYZE_STEP_PENALTY = 0.01
-SUBMIT_COMPLETION_BONUS = 0.30
-TIMEOUT_PENALTY = 0.12
+NO_PROGRESS_PENALTY = 0.20
+REPEATED_ACTION_PENALTY = 0.10
 VALID_ACTIONS = {"analyze_code", "edit_code", "run_tests", "submit_solution"}
 
 
@@ -78,7 +74,10 @@ class PythonCodeReviewEnvironment(
         self._done = False
         self._last_status = "Call reset() to start."
         self._last_reward = RewardDetails(value=0.0, reason="Environment initialized.")
-        self._reward_history: list[float] = []
+        self.reward_history: list[float] = []
+        self.previous_score = 0.0
+        self.last_code = ""
+        self._last_reward_components: dict[str, float] = {}
         self._metrics = self._blank_metrics()
         self._last_action_type = ""
 
@@ -100,7 +99,10 @@ class PythonCodeReviewEnvironment(
         self._task = task
         self._done = False
         self._metrics = self._blank_metrics()
-        self._reward_history = []
+        self.reward_history = []
+        self.previous_score = 0.0
+        self.last_code = ""
+        self._last_reward_components = {}
         self._last_action_type = ""
         self._last_status = "Inspect the code, run checks, edit the code, then submit."
         self._last_reward = RewardDetails(
@@ -161,11 +163,11 @@ class PythonCodeReviewEnvironment(
                 self._handle_edit(code)
             elif action_type == "submit_solution":
                 self._handle_scored_action(action_type=action_type, candidate_code=self._state.current_code, include_hidden=True)
-                self._done = True
             else:
                 self._apply_invalid_action(f"Unsupported action_type '{action_type}'.")
 
             self._state.attempts_remaining = max(self._task.max_steps - self._state.step_count, 0)
+            self._done = self._done or self._state.score >= 1.0
             if self._state.attempts_remaining == 0 and not self._done:
                 self._auto_submit()
 
@@ -226,70 +228,58 @@ class PythonCodeReviewEnvironment(
 
     def compute_reward(
         self,
-        action_type: str,
-        previous_metrics: dict[str, float],
-        current_metrics: dict[str, float],
-        grade: TaskGrade,
-        code_changed: bool,
-        invalid_action: bool = False,
-    ) -> RewardDetails:
-        """Compute a bounded dynamic reward with progress and efficiency shaping."""
-        prev_score = _clamp(previous_metrics.get("score", 0.0))
-        curr_score = _clamp(current_metrics.get("score", 0.0))
-        score_delta = curr_score - prev_score
-        test_delta = current_metrics.get("test_fraction", 0.0) - previous_metrics.get("test_fraction", 0.0)
-        syntax_delta = current_metrics.get("syntax_score", 0.0) - previous_metrics.get("syntax_score", 0.0)
-        quality_delta = current_metrics.get("quality_score", 0.0) - previous_metrics.get("quality_score", 0.0)
-
-        step_penalty = BASE_STEP_PENALTY + (ANALYZE_STEP_PENALTY if action_type == "analyze_code" else 0.0)
-        repeated_penalty = REPEATED_ACTION_PENALTY if action_type == self._last_action_type else 0.0
-        no_progress = (
-            score_delta <= 1e-9
-            and test_delta <= 1e-9
-            and syntax_delta <= 1e-9
-            and quality_delta <= 1e-9
-            and not code_changed
-        )
-        stagnation_penalty = NO_PROGRESS_PENALTY if no_progress and not invalid_action else 0.0
-        regression_penalty = max(-score_delta, 0.0) * 0.6 + repeated_penalty + step_penalty
-        invalid_penalty = INVALID_ACTION_PENALTY if invalid_action else 0.0
-        timeout_penalty = TIMEOUT_PENALTY if bool(grade.timed_out) else 0.0
-
-        progress_reward = max(score_delta, 0.0) * 0.7
-        syntax_reward = max(syntax_delta, 0.0) * 0.5
-        test_reward = max(test_delta, 0.0) * 1.0
-        quality_bonus = max(quality_delta, 0.0) * 0.2
-        correctness_bonus = SUBMIT_COMPLETION_BONUS if action_type == "submit_solution" and curr_score >= 0.999 else 0.0
-
-        reward_value = (
-            progress_reward
-            + syntax_reward
-            + test_reward
-            + quality_bonus
-            + correctness_bonus
+        old_code: str,
+        new_code: str,
+        prev_score: float,
+        curr_score: float,
+        test_results: dict[str, int],
+    ) -> float:
+        """Compute the requested bounded reward from score delta and action outcome."""
+        progress = curr_score - prev_score
+
+        passed = int(test_results["passed"])
+        total = int(test_results["total"])
+        test_ratio = passed / total if total > 0 else 0.0
+
+        try:
+            compile(new_code, "<string>", "exec")
+            syntax_score = 1.0
+        except Exception:
+            syntax_score = 0.0
+
+        stagnation_penalty = NO_PROGRESS_PENALTY if new_code.strip() == old_code.strip() else 0.0
+        regression_penalty = max(0.0, prev_score - curr_score)
+        repetition_penalty = REPEATED_ACTION_PENALTY if new_code == getattr(self, "last_code", "") else 0.0
+        length_penalty = 0.1 if len(new_code) > len(old_code) * 1.5 else 0.0
+
+        reward = (
+            0.4 * progress
+            + 0.3 * test_ratio
+            + 0.2 * syntax_score
             - stagnation_penalty
             - regression_penalty
-            - invalid_penalty
-            - timeout_penalty
-        )
-        reward_value = max(-1.0, min(1.0, round(reward_value, 6)))
-        reward_value = self._stabilize_reward(reward_value)
-        return RewardDetails(
-            value=reward_value,
-            syntax_reward=round(syntax_reward, 6),
-            test_reward=round(test_reward, 6),
-            quality_bonus=round(quality_bonus, 6),
-            correctness_bonus=round(correctness_bonus, 6),
-            progress_delta=round(progress_reward, 6),
-            stagnation_penalty=round(stagnation_penalty, 6),
-            regression_penalty=round(regression_penalty, 6),
-            invalid_action_penalty=round(invalid_penalty, 6),
-            timeout_penalty=round(timeout_penalty, 6),
-            reason=f"{action_type} reward computed safely",
-            prev_score=round(prev_score, 6),
-            curr_score=round(curr_score, 6),
-            code_changed=bool(code_changed),
+            - repetition_penalty
+            - length_penalty
         )
+        reward = max(-1.0, min(1.0, reward))
+
+        self.reward_history.append(round(reward, 6))
+        if len(self.reward_history) >= 3 and len(set(self.reward_history[-3:])) == 1:
+            import random
+
+            reward = max(-1.0, min(1.0, reward + random.uniform(-0.05, 0.05)))
+            self.reward_history[-1] = round(reward, 6)
+
+        self._last_reward_components = {
+            "syntax_reward": round(0.2 * syntax_score, 6),
+            "test_reward": round(0.3 * test_ratio, 6),
+            "progress_delta": round(0.4 * progress, 6),
+            "stagnation_penalty": round(stagnation_penalty, 6),
+            "regression_penalty": round(regression_penalty, 6),
+            "repetition_penalty": round(repetition_penalty, 6),
+            "length_penalty": round(length_penalty, 6),
+        }
+        return round(reward, 6)
 
     def _safe_task_order(self) -> list[str]:
         """Load deterministic task ids with a hard fallback."""
@@ -310,20 +300,6 @@ class PythonCodeReviewEnvironment(
             "quality_score": 0.0,
         }
 
-    def _stabilize_reward(self, reward_value: float) -> float:
-        """Break exact three-step reward plateaus without adding randomness."""
-        rounded_reward = round(reward_value, 6)
-        if len(self._reward_history) >= 2 and self._reward_history[-1] == self._reward_history[-2] == rounded_reward:
-            adjustment = 0.001 if self._state.step_count % 2 == 0 else -0.001
-            rounded_reward = round(max(-1.0, min(1.0, rounded_reward + adjustment)), 6)
-        return rounded_reward
-
-    def _record_reward(self, reward_value: float) -> None:
-        """Track recent rewards so repeated plateaus can be detected."""
-        self._reward_history.append(round(float(reward_value), 6))
-        if len(self._reward_history) > 8:
-            self._reward_history = self._reward_history[-8:]
-
     def _select_task(self, task_id: Optional[str]) -> TaskSpec:
         """Select the requested task or advance deterministically."""
         try:
@@ -401,27 +377,45 @@ class PythonCodeReviewEnvironment(
 
     def _handle_scored_action(self, action_type: str, candidate_code: str, include_hidden: bool) -> None:
         """Grade code, update state, and compute reward for a valid action."""
-        task = self._task or self._select_task(None)
-        previous_metrics = dict(self._metrics)
-        prior_code = self._state.current_code
-        code_changed = candidate_code.strip() != prior_code.strip()
-        if action_type == "edit_code":
-            self._state.current_code = candidate_code
-        grade = self._safe_grade(task=task, candidate_code=self._state.current_code, include_hidden=include_hidden)
+        old_code = self._state.current_code
+        new_code = candidate_code
+        curr_score, test_results, grade = self.run_tests(new_code, include_hidden=include_hidden)
         current_metrics = self._metrics_from_grade(grade)
+        reward_value = self.compute_reward(
+            old_code=old_code,
+            new_code=new_code,
+            prev_score=self.previous_score,
+            curr_score=curr_score,
+            test_results=test_results,
+        )
+        self._state.current_code = new_code
         self._apply_grade_to_state(grade, include_hidden=include_hidden)
-        self._last_reward = self.compute_reward(
-            action_type=action_type,
-            previous_metrics=previous_metrics,
-            current_metrics=current_metrics,
-            grade=grade,
-            code_changed=code_changed,
-            invalid_action=False,
+        self._last_reward = RewardDetails(
+            value=reward_value,
+            syntax_reward=self._last_reward_components.get("syntax_reward", 0.0),
+            test_reward=self._last_reward_components.get("test_reward", 0.0),
+            quality_bonus=0.0,
+            correctness_bonus=0.0,
+            progress_delta=self._last_reward_components.get("progress_delta", 0.0),
+            stagnation_penalty=self._last_reward_components.get("stagnation_penalty", 0.0),
+            regression_penalty=round(
+                self._last_reward_components.get("regression_penalty", 0.0)
+                + self._last_reward_components.get("repetition_penalty", 0.0)
+                + self._last_reward_components.get("length_penalty", 0.0),
+                6,
+            ),
+            invalid_action_penalty=0.0,
+            timeout_penalty=0.0,
+            reason=f"{action_type} reward computed safely",
+            prev_score=round(self.previous_score, 6),
+            curr_score=round(curr_score, 6),
+            code_changed=bool(new_code.strip() != old_code.strip()),
         )
         self._last_status = self._build_status(action_type, grade)
         self._metrics = current_metrics
+        self.previous_score = curr_score
+        self.last_code = new_code
         self._last_action_type = action_type
-        self._record_reward(self._last_reward.value)
         self._append_history(action_type, self._last_status, self._last_reward.value)
 
     def _handle_edit(self, code: Optional[str]) -> None:
@@ -434,18 +428,18 @@ class PythonCodeReviewEnvironment(
 
     def _apply_invalid_action(self, reason: str) -> None:
         """Record an invalid action without crashing the episode."""
-        previous_metrics = dict(self._metrics)
-        grade = TaskGrade(score=previous_metrics["score"], syntax_score=previous_metrics["syntax_score"])
-        self._last_reward = self.compute_reward(
-            action_type="invalid",
-            previous_metrics=previous_metrics,
-            current_metrics=previous_metrics,
-            grade=grade,
+        current_score = _clamp(self.previous_score)
+        reward_value = -INVALID_ACTION_PENALTY
+        self.reward_history.append(round(reward_value, 6))
+        self._last_reward = RewardDetails(
+            value=reward_value,
+            invalid_action_penalty=INVALID_ACTION_PENALTY,
+            reason=reason,
+            prev_score=current_score,
+            curr_score=current_score,
             code_changed=False,
-            invalid_action=True,
         )
         self._last_status = reason
-        self._record_reward(self._last_reward.value)
         self._append_history("analyze_code", reason, self._last_reward.value)
 
     def _auto_submit(self) -> None:
@@ -453,6 +447,7 @@ class PythonCodeReviewEnvironment(
         task = self._task or self._select_task(None)
         grade = self._safe_grade(task=task, candidate_code=self._state.current_code, include_hidden=True)
         self._apply_grade_to_state(grade, include_hidden=True)
+        self.previous_score = _clamp(grade.score)
         self._done = True
         self._state.done = True
         self._last_status = f"Auto-submitted. Final score: {_clamp(grade.score):.3f}"

server/requirements.txt

@@ -1,4 +1,5 @@
 fastapi>=0.115,<1.0
-uvicorn[standard]>=0.30,<1.0
+uvicorn>=0.30,<1.0
 pydantic>=2.0,<3.0
+openai>=1.0,<3.0
 pytest>=8.0,<9.0