simulator remaining and environment

.gitignore

@@ -2,4 +2,4 @@
 /trade_env/__pycache__
 /trade_env/env/__pycache__
 /trade_env/tests/__pycache__
-
+.env

README.md

@@ -4,4 +4,6 @@
 4. inference.py        
 5. run end-to-end      
 6. add PPO agent       
-7. improve logic       
+7. improve logic       
+
+`uvicorn trade_env.server.app:app --reload`

inference.py

@@ -1,5 +1,111 @@
-""" 
-    The main mandatory file which calls the env , logs 
-    
 """
+inference.py - must be in root directory
+Uses OpenAI client for LLM calls as per hackathon requirements
+Emits [START], [STEP], [END] structured logs
+"""
+from dotenv import load_dotenv
+load_dotenv()
+import os
+from openai import OpenAI
+from trade_env.env.coach_env import CoachEnv
+from trade_env.schemas.action import Action, ActionType
+
+TASK_NAME   = "trader-coach"
+BENCHMARK   = "coach-env"
+MODEL_NAME  = os.getenv("MODEL_NAME", "gpt-4o-mini")
+API_BASE    = os.getenv("API_BASE_URL", "https://api.openai.com/v1")
+HF_TOKEN    = os.getenv("HF_TOKEN", "")
+MAX_STEPS   = 20
+
+client = OpenAI(
+    api_key=os.getenv("OPENAI_API_KEY"),
+    base_url=API_BASE
+)
+
+
+def get_llm_action(state: dict) -> int:
+    prompt = f"""You are a trading behavior coach. Given this trader state:
+- timestep: {state['timestep']}
+- price: {state['price']:.2f}
+- position: {state['position']}
+- loss_streak: {state['loss_streak']}
+- pnl: {state['pnl']:.2f}
+
+Choose intervention (respond with single integer only):
+0 = NO (do nothing)
+1 = WARN (light nudge)
+2 = REDUCE (reduce position size)
+3 = EXIT (exit position)
+4 = COOLDOWN (force break)"""
+
+    response = client.chat.completions.create(
+        model=MODEL_NAME,
+        messages=[{"role": "user", "content": prompt}],
+        max_tokens=5,
+        temperature=0.0
+    )
+
+    raw = response.choices[0].message.content.strip()
+    try:
+        action = int(raw)
+        if action not in range(5):
+            action = 0
+    except ValueError:
+        action = 0
+    return action
+
+
+def log_start():
+    print(f"[START] task={TASK_NAME} env={BENCHMARK} model={MODEL_NAME}")
+
+
+def log_step(step, action, reward, done, error=None):
+    error_val = error if error else "null"
+    print(f"[STEP] step={step} action={action} reward={reward:.4f} done={str(done).lower()} error={error_val}")
+
+
+def log_end(success, steps, score, rewards):
+    rewards_str = ",".join(f"{r:.4f}" for r in rewards)
+    print(f"[END] success={str(success).lower()} steps={steps} score={score:.4f} rewards={rewards_str}")
+
+
+def main():
+    env = CoachEnv()
+    rewards = []
+    steps_taken = 0
+
+    log_start()
+
+    try:
+        state = env.reset()
+
+        for step in range(1, MAX_STEPS + 1):
+            action_idx = get_llm_action(state)
+            action = Action(action=ActionType(action_idx))
+
+            next_state, reward, done, info = env.step(action)
+
+            log_step(step, ActionType(action_idx).name, reward, done)
+
+            rewards.append(reward)
+            steps_taken = step
+            state = next_state
+
+            if done:
+                break
+
+        total_reward = sum(rewards)
+        score = max(0.0, min(1.0, (total_reward + 1.0) / 2.0))
+        success = score > 0.1
+
+    except Exception as e:
+        log_step(steps_taken + 1, "NO", 0.0, True, error=str(e))
+        success = False
+        score = 0.0
+        rewards = rewards or [0.0]
+
+    log_end(success, steps_taken, score, rewards)
+
 
+if __name__ == "__main__":
+    main()

openenv.yaml

@@ -1,7 +1,17 @@
-spec_version: 1
-name: trade_env
-type: space
-runtime: fastapi
-app: server.app:app
-port: 8000
-
+name: RetailTraderBehaviorCoach
+version: "1.0"
+state:
+  timestep: int
+  price: float
+  position: int
+  loss_streak: int
+  pnl: float
+actions:
+  - NO
+  - WARN
+  - REDUCE
+  - EXIT
+  - COOLDOWN
+endpoints:
+  reset: /reset
+  step: /step

trade_env/agent/ppo_agent.py

@@ -0,0 +1,115 @@
+"""Proximal policy Optimization(PPO)"""
+from trade_env.schemas.action import Action
+from trade_env.schemas.state import State
+from trade_env.schemas.step_response import StepResponse
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from torch.distributions import Categorical
+
+
+class ActorCritic(torch.nn.Module):
+    def __init__(self, state_dim, action_dim):
+        super().__init__()
+        self.shared = nn.Sequential(
+            nn.Linear(state_dim, 64),
+            nn.Tanh(),
+            nn.Linear(64, 64),
+            nn.Tanh(),
+        )
+        self.actor = nn.Linear(64, action_dim)
+        self.critic = nn.Linear(64, 1)
+
+    def forward(self, x):
+        x = self.shared(x)
+        return self.actor(x), self.critic(x)
+
+
+class PPOAgent(nn.Module):
+    def __init__(self, state_dim, action_dim, lr=3e-4, gamma=0.99, eps_clip=0.2):
+        super().__init__()
+        self.gamma = gamma
+        self.eps_clip = eps_clip
+        self.model = ActorCritic(state_dim, action_dim)
+        self.optimizer = optim.Adam(self.model.parameters(), lr=lr)
+        self._clear_memory()
+
+    def _clear_memory(self):
+        self.states = []
+        self.actions = []
+        self.log_probs = []
+        self.rewards = []
+        self.dones = []
+        self.values = []
+
+    def _state_to_tensor(self, state):
+        return torch.tensor(list(state.values()), dtype=torch.float32)
+
+    def select_action(self, state):
+        state_t = self._state_to_tensor(state)
+        with torch.no_grad():
+            logits, value = self.model(state_t)
+        
+        dist = Categorical(logits=logits)
+        action = dist.sample()
+
+        self.states.append(state_t)
+        self.actions.append(action)
+        self.log_probs.append(dist.log_prob(action))
+        self.values.append(value.squeeze())
+
+        return action.item()
+
+    def store_outcome(self, reward, done):
+        self.rewards.append(reward)
+        self.dones.append(done)
+
+    def _compute_returns(self):
+        returns = []
+        G = 0
+        for reward, done in zip(reversed(self.rewards), reversed(self.dones)):
+            if done:
+                G = 0
+            G = reward + self.gamma * G
+            returns.insert(0, G)
+        return torch.tensor(returns, dtype=torch.float32)
+
+    def update(self, epochs=4):
+        returns = self._compute_returns()
+
+        # detach everything collected during rollout
+        states = torch.stack(self.states).detach()
+        actions = torch.stack(self.actions).detach()
+        log_probs_old = torch.stack(self.log_probs).detach()
+        values_old = torch.stack(self.values).detach()
+
+        advantages = returns - values_old
+        # normalize advantages
+        advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8)
+
+        for _ in range(epochs):
+            logits, new_values = self.model(states)
+            dist = Categorical(logits=logits)
+            new_log_probs = dist.log_prob(actions)
+
+            ratio = torch.exp(new_log_probs - log_probs_old)
+
+            surr1 = ratio * advantages
+            surr2 = torch.clamp(ratio, 1 - self.eps_clip, 1 + self.eps_clip) * advantages
+
+            actor_loss = -torch.min(surr1, surr2).mean()
+            critic_loss = nn.MSELoss()(new_values.squeeze(), returns)
+            entropy_bonus = dist.entropy().mean()
+
+            loss = actor_loss + 0.5 * critic_loss - 0.01 * entropy_bonus
+
+            self.optimizer.zero_grad()
+            loss.backward()
+            torch.nn.utils.clip_grad_norm_(self.model.parameters(), 0.5)
+            self.optimizer.step()
+
+        self._clear_memory()
+
+if __name__ == "__main__":
+    agent = PPOAgent(state_dim=5, action_dim=5)
+    print("PPOAgent instantiated successfully.")

trade_env/env/coach_env.py

@@ -27,16 +27,6 @@ import random
 from enum import Enum
 from trade_env.schemas.action import ActionType, Action
 
-
-class Action(Enum):
-    
-    NO = 0
-    WARN = 1
-    REDUCE = 2
-    EXIT = 3
-    COOLDOWN = 4 #force stop for a tmframe 
-    
-
 class CoachEnv:
     
     def __init__(self):
@@ -46,6 +36,8 @@ class CoachEnv:
         self.pnl = 0
         self.loss_streak = 0
         self.pos = 0
+        self.entry_price = 100
+        
     
     def reset(self):
         """ resets the env
@@ -56,7 +48,7 @@ class CoachEnv:
         self.pnl = 0
         self.loss_streak = 0
         self.pos = 0
-        
+        self.entry_price = 100
         
         return self._get_state()
     
@@ -68,14 +60,14 @@ class CoachEnv:
         Args:
             action (): task for the agent to take given the sensor inputs in the env present
         """
-        action_type = action.action_type
+        action_type = action.action
         
         intr = 0
         if(action_type == ActionType.WARN):
             intr = .2
-        elif action_type == ActionType.REDUCE_SIZE:
+        elif action_type == ActionType.REDUCE:
             intr = 0.4
-        elif action_type == ActionType.EXIT_POSITION:
+        elif action_type == ActionType.EXIT:
             self.pos = 0
         elif action_type == ActionType.COOLDOWN:
             intr = 1.0
@@ -113,8 +105,8 @@ class CoachEnv:
         else:
             self.loss_streak = 0
 
-        reward = step_pnl - (0.1 * intr)
-
+        raw_reward = step_pnl - (0.1 * intr) - (0.5 * self.loss_streak if step_pnl < 0 else 0)
+        reward = max(-1.0, min(1.0, raw_reward / 50.0))         
         self.t += 1
         done = False
 
@@ -141,4 +133,4 @@ class CoachEnv:
             "position": self.pos,
             "loss_streak": self.loss_streak,
             "pnl": self.pnl
-        }
+        }   

trade_env/schemas/state.py

@@ -13,4 +13,5 @@ class State(BaseModel):
     price: float           
     position: int          
     loss_streak: int      
-    pnl: float             
+    pnl: float   
+    

trade_env/server/app.py

@@ -4,57 +4,38 @@ fast api endpoints which will be an HTTP server
 
 """
 
-try:
-    from openenv.core.env_server.http_server import create_app
-except Exception as e:  # pragma: no cover
-    raise ImportError(
-        "openenv is required for the web interface. Install dependencies with '\n    uv sync\n'"
-    ) from e
+from fastapi import FastAPI
+import uvicorn
+from trade_env.env.coach_env import CoachEnv
+from trade_env.schemas.action import Action
+from trade_env.schemas.state import State
+from trade_env.schemas.step_response import StepResponse
 
-try:
-    from ..models import TradeAction, TradeObservation
-    from .environment import TradeEnvironment
-except ModuleNotFoundError:
-    from models import TradeAction, TradeObservation
-    from trade_env.server.environment import TradeEnvironment
+app = FastAPI()
 
+env = CoachEnv()
 
-# Create the app with web interface and README integration
-app = create_app(
-    TradeEnvironment,
-    TradeAction,
-    TradeObservation,
-    env_name="trade_env",
-    max_concurrent_envs=1,  # increase this number to allow more concurrent WebSocket sessions
-)
 
+@app.post("/reset",response_model=State)
+def reset():
+    state = env.reset()    
+    return State(**state)   
 
-def main(host: str = "0.0.0.0", port: int = 8000):
-    """
-    Entry point for direct execution via uv run or python -m.
+@app.post("/step", response_model=StepResponse)
+def step(action: Action):
+    next_state, reward, done, info = env.step(action)
 
-    This function enables running the server without Docker:
-        uv run --project . server
-        uv run --project . server --port 8001
-        python -m trade_env.server.app
-
-    Args:
-        host: Host address to bind to (default: "0.0.0.0")
-        port: Port number to listen on (default: 8000)
-
-    For production deployments, consider using uvicorn directly with
-    multiple workers:
-        uvicorn trade_env.server.app:app --workers 4
-    """
-    import uvicorn
-
-    uvicorn.run(app, host=host, port=port)
+    return StepResponse(
+        next_state=State(**next_state),
+        reward=reward,
+        done=done,
+        info=info
+    )
 
+def main():
+    uvicorn.run("server.app:app", host="0.0.0.0", port=8000)
 
 if __name__ == "__main__":
-    import argparse
+    main()
 
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--port", type=int, default=8000)
-    args = parser.parse_args()
-    main(port=args.port)
+    

train.py

@@ -0,0 +1,21 @@
+# train.py
+from trade_env.env.coach_env import CoachEnv
+from trade_env.schemas.action import Action, ActionType
+from trade_env.agent.ppo_agent import PPOAgent
+
+env = CoachEnv()
+agent = PPOAgent(state_dim=5, action_dim=5)
+
+for episode in range(2000):
+    state = env.reset()
+    done = False
+
+    while not done:
+        action_idx = agent.select_action(state)
+        action = Action(action=ActionType(action_idx))
+        next_state, reward, done, info = env.step(action)
+        agent.store_outcome(reward, done)
+        state = next_state
+
+    agent.update()
+    print(f"Ep {episode} | PnL: {info['pnl']:.2f} | Action: {action_idx} | Trader: {info['trader_action']}")