Fix: Balance task difficulty

env/core.py

@@ -42,12 +42,12 @@ TASKS = {
         difficulty="easy",
         description="Optimize this 3-server cluster. Start by analyzing load patterns, then iteratively adjust each server. Final reward requires ALL servers properly sized.",
         initial_resources=[
-            {"id": "srv-1", "type": "m5.xlarge", "cpu_usage": 15.0, "mem_usage": 10.0, "monthly_cost": 140.0},
-            {"id": "srv-2", "type": "m5.xlarge", "cpu_usage": 12.0, "mem_usage": 8.0, "monthly_cost": 140.0},
-            {"id": "srv-3", "type": "m5.xlarge", "cpu_usage": 18.0, "mem_usage": 12.0, "monthly_cost": 140.0},
+            {"id": "srv-1", "type": "m5.large", "cpu_usage": 20.0, "mem_usage": 15.0, "monthly_cost": 70.0},
+            {"id": "srv-2", "type": "m5.large", "cpu_usage": 18.0, "mem_usage": 12.0, "monthly_cost": 70.0},
+            {"id": "srv-3", "type": "m5.large", "cpu_usage": 22.0, "mem_usage": 16.0, "monthly_cost": 70.0},
         ],
-        sla={"max_latency_ms": 100.0, "max_budget": 80.0, "min_uptime_pct": 99.0},
-        load=45.0
+        sla={"max_latency_ms": 120.0, "max_budget": 80.0, "min_uptime_pct": 99.0},
+        load=30.0
     ),
     "medium": TaskConfig(
         task_id="medium_latency_fix",
@@ -55,12 +55,12 @@ TASKS = {
         difficulty="medium",
         description="Performance bottleneck! This cluster is struggling. Analyze each server's load, then iteratively upgrade undersized servers. Requires 4+ successful changes for max reward.",
         initial_resources=[
-            {"id": "srv-1", "type": "t3.nano", "cpu_usage": 95.0, "mem_usage": 85.0, "monthly_cost": 3.6},
-            {"id": "srv-2", "type": "t3.nano", "cpu_usage": 88.0, "mem_usage": 80.0, "monthly_cost": 3.6},
-            {"id": "srv-3", "type": "t3.nano", "cpu_usage": 92.0, "mem_usage": 83.0, "monthly_cost": 3.6},
+            {"id": "srv-1", "type": "t3.small", "cpu_usage": 90.0, "mem_usage": 80.0, "monthly_cost": 11.5},
+            {"id": "srv-2", "type": "t3.small", "cpu_usage": 85.0, "mem_usage": 75.0, "monthly_cost": 11.5},
+            {"id": "srv-3", "type": "t3.small", "cpu_usage": 88.0, "mem_usage": 78.0, "monthly_cost": 11.5},
         ],
-        sla={"max_latency_ms": 80.0, "max_budget": 100.0, "min_uptime_pct": 99.9},
-        load=30.0
+        sla={"max_latency_ms": 100.0, "max_budget": 100.0, "min_uptime_pct": 99.9},
+        load=15.0
     ),
     "hard": TaskConfig(
         task_id="hard_balance",
@@ -68,14 +68,14 @@ TASKS = {
         difficulty="hard",
         description="Tight budget constraint! Optimize a mixed 5-server cluster. Must achieve optimal cost-efficiency while maintaining performance. Requires 5+ iterative changes, exploring different configurations.",
         initial_resources=[
-            {"id": "srv-1", "type": "m5.xlarge", "cpu_usage": 8.0, "mem_usage": 6.0, "monthly_cost": 140.0},
-            {"id": "srv-2", "type": "m5.xlarge", "cpu_usage": 10.0, "mem_usage": 8.0, "monthly_cost": 140.0},
+            {"id": "srv-1", "type": "m5.large", "cpu_usage": 15.0, "mem_usage": 10.0, "monthly_cost": 70.0},
+            {"id": "srv-2", "type": "m5.large", "cpu_usage": 12.0, "mem_usage": 8.0, "monthly_cost": 70.0},
             {"id": "srv-3", "type": "t3.nano", "cpu_usage": 95.0, "mem_usage": 90.0, "monthly_cost": 3.6},
             {"id": "srv-4", "type": "t3.nano", "cpu_usage": 98.0, "mem_usage": 92.0, "monthly_cost": 3.6},
             {"id": "srv-5", "type": "t3.medium", "cpu_usage": 45.0, "mem_usage": 40.0, "monthly_cost": 23.0},
         ],
         sla={"max_latency_ms": 100.0, "max_budget": 60.0, "min_uptime_pct": 99.9},
-        load=50.0
+        load=35.0
     ),
 }
 
@@ -156,6 +156,8 @@ class CloudOpsEnvironment:
         
         message = self._parse_and_execute(msg)
         
+        self._ep.last_action_success = message.startswith("Changed")
+        
         new_cost = sum(r.monthly_cost for r in self._ep.resources)
         latency, error_rate, utilization = self._calculate_metrics(
             self._ep.current_load, 
@@ -178,8 +180,8 @@ class CloudOpsEnvironment:
         reward = self._calculate_iterative_reward(latency, error_rate, new_cost, prev_cost, prev_latency)
         
         done = (
-            reward.value >= 0.98 and self._ep.changes_made >= 3 or
-            self._ep.steps >= self._max_steps
+            self._ep.steps >= self._max_steps or
+            (self._ep.changes_made >= 3 and reward.value >= 0.95)
         )
         
         obs = self._build_observation(message)
@@ -232,19 +234,19 @@ class CloudOpsEnvironment:
         cost_improvement = (prev_cost - new_cost) / (prev_cost + 1e-6)
         latency_improvement = (prev_latency - latency) / (prev_latency + 1e-6)
         
-        change_bonus = min(self._ep.changes_made * 0.08, 0.4)
+        change_bonus = min(self._ep.changes_made * 0.06, 0.3)
         
         cost_ratio = new_cost / budget
-        cost_reward = 0.25 * (1.0 / (1.0 + max(0, cost_ratio - 1)))
+        cost_reward = 0.3 * (1.0 / (1.0 + max(0, cost_ratio - 1)))
         
         lat_ratio = latency / max_latency
-        perf_reward = 0.25 * (1.0 / (1.0 + max(0, lat_ratio - 1)))
+        perf_reward = 0.3 * (1.0 / (1.0 + max(0, lat_ratio - 1)))
         
         improvement_bonus = 0.0
         if cost_improvement > 0:
-            improvement_bonus += min(cost_improvement * 0.2, 0.15)
+            improvement_bonus += min(cost_improvement * 0.15, 0.1)
         if latency_improvement > 0:
-            improvement_bonus += min(latency_improvement * 0.2, 0.15)
+            improvement_bonus += min(latency_improvement * 0.15, 0.1)
         
         base_reward = cost_reward + perf_reward
         total_reward = min(1.0, base_reward + change_bonus + improvement_bonus)
@@ -252,8 +254,9 @@ class CloudOpsEnvironment:
         if error_rate > 0.1:
             total_reward *= (1.0 - error_rate)
         
-        if self._ep.changes_made >= 3 and cost_reward > 0.2 and perf_reward > 0.2:
-            total_reward = min(1.0, total_reward + 0.1)
+        exploration_bonus = min(self._ep.steps * 0.03, 0.15)
+        if self._ep.last_action_success:
+            total_reward = min(1.0, total_reward + exploration_bonus)
         
         initial_latency = self._ep.initial_latency
         initial_cost = self._ep.initial_cost