Upload folder using huggingface_hub

Dockerfile

@@ -0,0 +1,69 @@
+ARG BASE_IMAGE=ghcr.io/meta-pytorch/openenv-base:latest
+FROM ${BASE_IMAGE} AS builder
+
+WORKDIR /app
+
+# Ensure git is available (required for installing dependencies from VCS)
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends git && \
+    rm -rf /var/lib/apt/lists/*
+
+# Build argument to control whether we're building standalone or in-repo
+ARG BUILD_MODE=in-repo
+ARG ENV_NAME=app
+
+# Copy environment code (always at root of build context)
+COPY . /app/env
+
+# For in-repo builds, openenv is already vendored in the build context
+# For standalone builds, openenv will be installed via pyproject.toml
+WORKDIR /app/env
+
+# Ensure uv is available (for local builds where base image lacks it)
+RUN if ! command -v uv >/dev/null 2>&1; then \
+        curl -LsSf https://astral.sh/uv/install.sh | sh && \
+        mv /root/.local/bin/uv /usr/local/bin/uv && \
+        mv /root/.local/bin/uvx /usr/local/bin/uvx; \
+    fi
+    
+# Install dependencies using uv sync
+# If uv.lock exists, use it; otherwise resolve on the fly
+RUN --mount=type=cache,target=/root/.cache/uv \
+    if [ -f uv.lock ]; then \
+        uv sync --frozen --no-install-project --no-editable; \
+    else \
+        uv sync --no-install-project --no-editable; \
+    fi
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    if [ -f uv.lock ]; then \
+        uv sync --frozen --no-editable; \
+    else \
+        uv sync --no-editable; \
+    fi
+
+# Final runtime stage
+FROM ${BASE_IMAGE}
+
+WORKDIR /app
+
+# Copy the virtual environment from builder
+COPY --from=builder /app/env/.venv /app/.venv
+
+# Copy the environment code
+COPY --from=builder /app/env /app/env
+
+# Set PATH to use the virtual environment
+ENV PATH="/app/.venv/bin:$PATH"
+
+# Set PYTHONPATH so imports work correctly
+ENV PYTHONPATH="/app/env:$PYTHONPATH"
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
+    CMD curl -f http://localhost:8000/health || exit 1
+
+# Run the FastAPI server
+# The module path is constructed to work with the /app/env structure
+ENV ENABLE_WEB_INTERFACE=true
+CMD ["sh", "-c", "cd /app/env && uvicorn server.app:app --host 0.0.0.0 --port 8000"]

README.md

@@ -1,10 +1,7 @@
 ---
-title: App
-emoji: 🦀
-colorFrom: blue
-colorTo: indigo
-sdk: docker
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+title: Object Placement
+emoji: 🔊
+colorFrom: purple
+colorTo: yellow
+base_path: /web
+---

__init__.py

@@ -0,0 +1,16 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""App Environment."""
+
+from .client import AppEnv
+from .models import AppAction, AppObservation
+
+__all__ = [
+    "AppAction",
+    "AppObservation",
+    "AppEnv",
+]

client.py

@@ -0,0 +1,51 @@
+"""App Environment Client."""
+
+from typing import Dict
+
+from openenv.core import EnvClient
+from openenv.core.client_types import StepResult
+
+from .models import AppAction, AppObservation, AppState
+
+
+class AppEnv(EnvClient[AppAction, AppObservation, AppState]):
+
+    def _step_payload(self, action: AppAction) -> Dict:
+
+        return {
+            "placement": action.placement,
+            "isSegmentation": action.isSegmentation,
+            "findObjects": action.findObjects,
+        }
+
+    def _parse_result(self, payload: Dict) -> StepResult[AppObservation]:
+
+        obs_data = payload.get("observation", {})
+        observation = AppObservation(
+            currentGrid=obs_data.get("currentGrid", []),
+            positions=obs_data.get("positions", {}),
+            objectsLeft=obs_data.get("objectsLeft", []),
+            objectsFound=obs_data.get("objectsFound", []),
+            reward=obs_data.get("reward", 0.0),
+            isDone=obs_data.get("isDone", False),
+        )
+
+        return StepResult(
+            observation=observation,
+            reward=payload.get("reward"),
+            done=payload.get("done", obs_data.get("isDone", False)),
+        )
+
+    def _parse_state(self, payload: Dict) -> AppState:
+
+        return AppState(
+            episode_id=payload.get("episode_id"),
+            step_count=payload.get("step_count", 0),
+            currentGrid=payload.get("currentGrid", []),
+            weightedGrid=payload.get("weightedGrid", []),
+            reward=payload.get("reward", 0.0),
+            isDone=payload.get("isDone", False),
+            objectsLeft=payload.get("objectsLeft", []),
+            objectsFound=payload.get("objectsFound", []),
+            ObjectsPresent=payload.get("ObjectsPresent", {}),
+        )

inference.py

@@ -0,0 +1,41 @@
+import os
+import re
+import base64
+import textwrap
+from io import BytesIO
+from typing import List, Optional, Dict
+
+from openai import OpenAI
+import numpy as np
+from PIL import Image
+
+
+API_BASE_URL = os.getenv("API_BASE_URL")
+API_KEY = os.getenv("HF_TOKEN") or os.getenv("API_KEY")
+MODEL_NAME = os.getenv("MODEL_NAME")
+
+SYSTEM_PROMPT = textwrap.dedent(
+    """
+    You control a web browser through BrowserGym.
+    Reply with exactly one action string.
+    The action must be a valid BrowserGym command such as:
+    - noop()
+    - click('<BID>')
+    - type('selector', 'text to enter')
+    - fill('selector', 'text to enter')
+    - send_keys('Enter')
+    - scroll('down')
+    Use single quotes around string arguments.
+    When clicking, use the BrowserGym element IDs (BIDs) listed in the user message.
+    If you are unsure, respond with noop().
+    Do not include explanations or additional text.
+    """
+).strip()
+
+
+def main() -> None:
+    client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
+
+
+if __name__ == "__main__":
+    main()

models.py

@@ -0,0 +1,94 @@
+from openenv.core.env_server.types import Action, Observation, State
+from pydantic import Field
+from typing import List, Dict, Tuple
+
+
+class AppAction(Action):
+    """Action for the App environment"""
+
+    placement: Dict[str, Tuple[int, int, int, bool]] = Field(
+        default_factory=dict, description="Placement of the object in a 3D grid"
+    )
+    isSegmentation: bool = Field(
+        default=True, description="Whether the model is segmenting the objects"
+    )
+    findObjects: Dict[str, Tuple[int, int, int, bool]] = Field(
+        default_factory=dict, description="Dictionary of objects"
+    )
+
+
+class AppObservation(Observation):
+    """Observation from the App environment"""
+
+    currentGrid: List[List[List[int]]] = Field(
+        default_factory=list,
+        description="Current placement of the objects in a 3D grid",
+    )
+    positions: Dict[str, Tuple[int, int, int, bool]] = Field(
+        default_factory=dict,
+        description="Dictionary of objects with their positions in the environment",
+    )
+    objectsLeft: List[str] = Field(
+        default_factory=list,
+        description="List of unorganised objects left in the environment",
+    )
+    objectsFound: List[str] = Field(
+        default_factory=list,
+        description="List of objects found in the environment",
+    )
+    reward: float = Field(
+        default=0.0, description="Reward received after taking the action"
+    )
+    isDone: bool = Field(default=False, description="Whether the episode has ended")
+
+    rewardFeedback: list[str] = Field(
+        default_factory=list,
+        description="List of feedback strings describing the reward received after taking the action",
+    )
+
+    rewardList: list[float] = Field(
+        default_factory=list,
+        description="List of reward values received after taking the action",
+    )
+
+
+class AppState(State):
+    """State for the App environment"""
+
+    currentGrid: List[List[List[int]]] = Field(
+        default_factory=list,
+        description="Initial state of the environment with unorganised objects",
+    )
+
+    weightedGrid: List[List[List[float]]] = Field(
+        default_factory=list,
+        description="Weighted grid used when scoring placements",
+    )
+
+    objectsLeft: List[str] = Field(
+        default_factory=list,
+        description="List of unorganised objects left in the environment",
+    )
+    objectsFound: List[str] = Field(
+        default_factory=list,
+        description="List of objects found in the environment",
+    )
+    reward: float = Field(
+        default=0.0, description="Reward received after taking the action"
+    )
+    isDone: bool = Field(default=False, description="Whether the episode has ended")
+
+    ObjectsPresent: Dict[str, Tuple[int, int, int, bool]] = Field(
+        default_factory=dict,
+        description="Placed objects and their current positions in the environment",
+    )
+
+    rewardFeedback: list[str] = Field(
+        default_factory=list,
+        description="List of feedback strings describing the reward received after taking the action",
+    )
+
+    rewardList: list[float] = Field(
+        default_factory=list,
+        description="List of reward values received after taking the action",
+    )

openenv.yaml

@@ -0,0 +1,6 @@
+spec_version: 1
+name: app
+type: space
+runtime: fastapi
+app: server.app:app
+port: 8000

pyproject.toml

@@ -0,0 +1,45 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+[build-system]
+requires = ["setuptools>=45", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "openenv-app"
+version = "0.1.0"
+description = "App environment for OpenEnv"
+requires-python = ">=3.10"
+dependencies = [
+    # Core OpenEnv runtime (provides FastAPI server + HTTP client types)
+    # install from github
+    # "openenv-core[core] @ git+https://github.com/meta-pytorch/OpenEnv.git",
+    "openenv-core[core]>=0.2.1",
+    # Environment-specific dependencies
+    # Add all dependencies needed for your environment here
+    # Examples:
+    # "numpy>=1.19.0",
+    # "torch>=2.0.0",
+    # "gymnasium>=0.29.0",
+    # "openspiel>=1.0.0",
+    # "smolagents>=1.22.0,<2",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0.0",
+    "pytest-cov>=4.0.0",
+]
+
+[project.scripts]
+# Server entry point - enables running via: uv run --project . server
+# or: python -m app.server.app
+server = "app.server.app:main"
+
+[tool.setuptools]
+include-package-data = true
+packages = ["app", "app.server"]
+package-dir = { "app" = ".", "app.server" = "server" }

server/__init__.py

@@ -0,0 +1,11 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""App environment server components."""
+
+from .app_environment import AppEnvironment
+
+__all__ = ["AppEnvironment"]

server/app.py

@@ -0,0 +1,53 @@
+try:
+    from openenv.core.env_server.http_server import create_app
+except Exception as e:  # pragma: no cover
+    raise ImportError(
+        "openenv is required for the web interface. Install dependencies with '\n    uv sync\n'"
+    ) from e
+
+try:
+    from app.models import AppAction, AppObservation
+    from app.server.app_environment import AppEnvironment
+except ModuleNotFoundError:
+    from models import AppAction, AppObservation
+    from server.app_environment import AppEnvironment
+
+
+app = create_app(
+    AppEnvironment,
+    AppAction,
+    AppObservation,
+    env_name="app",
+    max_concurrent_envs=1,  # increase this number to allow more concurrent WebSocket sessions
+)
+
+
+def main(host: str = "0.0.0.0", port: int = 8000):
+    """
+    Entry point for direct execution via uv run or python -m.
+
+    This function enables running the server without Docker:
+        uv run --project . server
+        uv run --project . server --port 8001
+        python -m app.server.app
+
+    Args:
+        host: Host address to bind to (default: "0.0.0.0")
+        port: Port number to listen on (default: 8000)
+
+    For production deployments, consider using uvicorn directly with
+    multiple workers:
+        uvicorn app.server.app:app --workers 4
+    """
+    import uvicorn
+
+    uvicorn.run(app, host=host, port=port)
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--port", type=int, default=8000)
+    args = parser.parse_args()
+    main(port=args.port)

server/app_environment.py

@@ -0,0 +1,96 @@
+from uuid import uuid4
+
+from openenv.core.env_server.interfaces import Environment
+
+try:
+    from ..models import AppAction, AppObservation, AppState
+except ImportError:
+    from models import AppAction, AppObservation, AppState
+
+try:
+    from ..utils import *
+except ImportError:
+    from utils import *
+
+
+class AppEnvironment(Environment):
+
+    SUPPORTS_CONCURRENT_SESSIONS: bool = True
+
+    def __init__(self):
+        self._state = self._new_state()
+        self._reset_count = 0
+
+    def _new_state(self) -> AppState:
+        grid, placed = initGrid()
+
+        return AppState(
+            episode_id=str(uuid4()),
+            step_count=0,
+            currentGrid=grid,
+            weightedGrid=initWeightedGrid(),
+            objectsLeft=list(OBJECTS.keys()),
+            objectsFound=[],
+            reward=0.0,
+            isDone=False,
+            ObjectsPresent=placed,
+            rewardFeedback=[],
+            rewardList=[],
+        )
+
+    def reset(self) -> AppObservation:
+        self._state = self._new_state()
+
+        return AppObservation(
+            currentGrid=self._state.currentGrid,
+            positions=self._state.ObjectsPresent,
+            objectsLeft=self._state.objectsLeft,
+            objectsFound=self._state.objectsFound,
+            reward=self._state.reward,
+            isDone=self._state.isDone,
+            rewardFeedback=self._state.rewardFeedback,
+            rewardList=self._state.rewardList,
+        )
+
+    def step(self, action: AppAction) -> AppObservation:
+        if not isinstance(self._state, AppState):
+            self._state = self._new_state()
+
+        self._state.step_count += 1
+
+        reward = 0.0
+        if action.isSegmentation:
+            reward += 10.0
+            appendRewardFeedback(self._state, "Segmentation successful.", reward)
+
+        if action.placement:
+            reward += place(action.isSegmentation, action.placement, self._state)
+            appendRewardFeedback(self._state, "Object placed successfully.", reward)
+
+        if action.findObjects:
+            reward += findobject(action.isSegmentation, action.findObjects, self._state)
+            appendRewardFeedback(self._state, "Object found successfully.", reward)
+
+        if len(self._state.objectsLeft) == 0:
+            self._state.isDone = True
+            reward += 10.0
+            appendRewardFeedback(
+                self._state, "All objects found. Episode completed!", reward
+            )
+
+        self._state.reward += reward / (10**self._state.step_count)
+
+        return AppObservation(
+            currentGrid=self._state.currentGrid,
+            positions=self._state.ObjectsPresent,
+            objectsLeft=self._state.objectsLeft,
+            objectsFound=self._state.objectsFound,
+            reward=self._state.reward,
+            isDone=self._state.isDone,
+            rewardFeedback=self._state.rewardFeedback,
+            rewardList=self._state.rewardList,
+        )
+
+    @property
+    def state(self) -> dict:
+        return self._state.model_dump()

server/requirements.txt

@@ -0,0 +1,6 @@
+openenv-core
+fastapi
+uvicorn
+numpy
+scikit-learn
+matplotlib

utils.py

@@ -0,0 +1,294 @@
+from matplotlib.pylab import randint
+from numpy import ones, zeros, random
+from sklearn.metrics import mean_squared_error as MSE
+
+
+random.seed(123)
+
+OBJECTS = {
+    "book": {"dims": [4, 4, 2], "stack": True},
+    "penstand": {"dims": [2, 2, 4], "stack": True},
+    "bottle": {"dims": [2, 2, 6], "stack": False},
+    "pen": {"dims": [1, 1, 4], "stack": False},
+    "pencil": {"dims": [1, 1, 6], "stack": False},
+    "eraser": {"dims": [2, 1, 1], "stack": False},
+    "powerbank": {"dims": [4, 2, 1], "stack": False},
+    "mobile": {"dims": [4, 2, 1], "stack": False},
+    "laptop": {"dims": [6, 4, 1], "stack": True},
+    "monitor": {"dims": [6, 4, 2], "stack": False},
+    "keyboard": {"dims": [6, 2, 1], "stack": False},
+    "mouse": {"dims": [4, 2, 1], "stack": False},
+    "headphones": {"dims": [4, 4, 2], "stack": False},
+    "charger": {"dims": [2, 2, 1], "stack": False},
+    "notebook": {"dims": [4, 4, 1], "stack": True},
+    "folder": {"dims": [4, 4, 1], "stack": True},
+    "backpack": {"dims": [6, 4, 2], "stack": False},
+    "pouch": {"dims": [4, 4, 2], "stack": False},
+}
+
+OBJECT_NAMES = [
+    "book",
+    "penstand",
+    "bottle",
+    "pen",
+    "pencil",
+    "eraser",
+    "powerbank",
+    "mobile",
+    "laptop",
+    "monitor",
+    "keyboard",
+    "mouse",
+    "headphones",
+    "charger",
+    "notebook",
+    "folder",
+    "backpack",
+    "pouch",
+]
+
+
+def appendRewardFeedback(state, feedback, reward):
+    state.rewardFeedback.append(feedback)
+    state.rewardList.append(reward)
+
+
+def initDimentions(obj):
+    dims = obj.get("dims")
+    if dims is None:
+        return []
+
+    return ones(dims, dtype=int).tolist()
+
+
+def initGrid():
+    sizeX, sizeY, sizeZ = randint(8, 12), randint(8, 12), randint(8, 12)
+    grid = zeros((sizeX, sizeY, sizeZ), dtype=int).tolist()
+
+    numObjs = randint(3, len(OBJECT_NAMES) + 1)
+    chosenNames = random.choice(OBJECT_NAMES, size=numObjs, replace=False)
+
+    placed = {}
+
+    for name in chosenNames:
+        obj = OBJECTS.get(name)
+
+        dimX, dimY, dimZ = obj["dims"]
+
+        if dimX > sizeX or dimY > sizeY or dimZ > sizeZ:
+            continue
+
+        isPlaced = False
+        tryPlaced = 0
+
+        while not isPlaced and tryPlaced < 100:
+            posX = randint(0, sizeX - dimX + 1)
+            posY = randint(0, sizeY - dimY + 1)
+            posZ = 0
+
+            canPlace = True
+            for i in range(dimX):
+                for j in range(dimY):
+                    for k in range(dimZ):
+                        if (
+                            grid[posX + i][posY + j][posZ + k] != 0
+                            and obj["stack"] == False
+                        ):
+                            canPlace = False
+                            break
+                        else:
+                            canPlace = True
+                    if not canPlace:
+                        break
+                if not canPlace:
+                    break
+
+            if canPlace:
+                for i in range(dimX):
+                    for j in range(dimY):
+                        for k in range(dimZ):
+                            if (
+                                obj["stack"]
+                                and grid[posX + i][posY + j][posZ + k] > 0
+                                and posZ + k + 1 < sizeZ
+                            ):
+                                grid[posX + i][posY + j][posZ + k + 1] += 1
+                            else:
+                                grid[posX + i][posY + j][posZ + k] += 1
+
+            placed[name] = (posX, posY, posZ, obj["stack"])
+            isPlaced = True
+
+    return (grid, placed)
+
+
+def initWeightedGrid():
+    grid = random.uniform(0, 1, (randint(5, 11), randint(5, 11), randint(5, 11)))
+
+    x_mid = grid.shape[0] // 2
+    x_span = grid.shape[0] // 4
+    y_front = grid.shape[1] // 3
+
+    grid[x_mid - x_span : x_mid + x_span, :y_front, :] *= 0.2
+
+    return grid
+
+
+def place(segment, objects, state):
+    dims = state.currentGrid
+    weight = state.weightedGrid
+    reward = 0.0
+    totalObjs = len(objects)
+    reward_per_obj_placed = 45.0 / totalObjs
+
+    if segment:
+        appendRewardFeedback(
+            state, "Placing objects without segmentation is not allowed.", -60.0
+        )
+        return -60.0
+
+    for obj_name, pos in objects.items():
+        obj = OBJECTS.get(obj_name)
+        if obj is None:
+            appendRewardFeedback(
+                state, f"Object '{obj_name}' is not recognized.", -reward_per_obj_placed
+            )
+            reward -= reward_per_obj_placed
+            continue
+
+        objGrid = initDimentions(obj)
+        placement_failed = False
+
+        for i in range(len(objGrid)):
+            for j in range(len(objGrid[0])):
+                for k in range(len(objGrid[0][0])):
+                    if (
+                        pos[0] + i >= len(dims)
+                        or pos[1] + j >= len(dims[0])
+                        or pos[2] + k >= len(dims[0][0])
+                    ):
+                        reward -= reward_per_obj_placed
+                        appendRewardFeedback(
+                            state,
+                            f"Object '{obj_name}' placement is out of bounds.",
+                            -reward_per_obj_placed,
+                        )
+                        placement_failed = True
+                        break
+
+                    if dims[pos[0] + i][pos[1] + j][pos[2] + k] > 0 and pos[3] == False:
+                        reward -= reward_per_obj_placed
+                        appendRewardFeedback(
+                            state,
+                            f"Object '{obj_name}' placement overlaps with another object and stacking is not allowed.",
+                            -reward_per_obj_placed,
+                        )
+                        placement_failed = True
+                        break
+
+                    elif (
+                        dims[pos[0] + i][pos[1] + j][pos[2] + k] > 0 and pos[3] == True
+                    ):
+                        if pos[2] + k + 1 <= len(objGrid[0][0]):
+                            dims[pos[0] + i][pos[1] + j][pos[2] + k + 1] += 1
+                            reward += (
+                                weight[pos[0] + i][pos[1] + j][pos[2] + k + 1]
+                                * reward_per_obj_placed
+                            )
+                            appendRewardFeedback(
+                                state,
+                                f"Object '{obj_name}' placed with stacking. Bonus: {weight[pos[0] + i][pos[1] + j][pos[2] + k + 1] * reward_per_obj_placed:.2f}",
+                                weight[pos[0] + i][pos[1] + j][pos[2] + k + 1]
+                                * reward_per_obj_placed,
+                            )
+                        else:
+                            reward -= reward_per_obj_placed
+                            appendRewardFeedback(
+                                state,
+                                f"Object '{obj_name}' placement failed. No space for stacking.",
+                                -reward_per_obj_placed,
+                            )
+                            placement_failed = True
+
+                        break
+
+                    else:
+                        dims[pos[0] + i][pos[1] + j][pos[2] + k] = 1
+                        reward += (
+                            reward_per_obj_placed
+                            * weight[pos[0] + i][pos[1] + j][pos[2] + k]
+                        )
+                        appendRewardFeedback(
+                            state,
+                            f"Object '{obj_name}' placed successfully. Bonus: {weight[pos[0] + i][pos[1] + j][pos[2] + k] * reward_per_obj_placed:.2f}",
+                            weight[pos[0] + i][pos[1] + j][pos[2] + k]
+                            * reward_per_obj_placed,
+                        )
+                if placement_failed:
+                    break
+            if placement_failed:
+                break
+
+        if not placement_failed:
+            state.ObjectsPresent[obj_name] = pos
+
+    return reward
+
+
+def findobject(segment, objects, state):
+
+    if not segment:
+        appendRewardFeedback(
+            state, "Finding objects without segmentation is not allowed.", -60.0
+        )
+        return -60.0
+
+    reward = 0.0
+    glMetric = 45.0 / len(state.ObjectsPresent)
+    objs = []
+    for obj_found, pos_found in objects.items():
+        pos_real = state.ObjectsPresent.get(obj_found)
+        if pos_real is None:
+            reward -= glMetric
+            appendRewardFeedback(
+                state, f"Object '{obj_found}' not found in the environment.", -glMetric
+            )
+            continue
+
+        if pos_found == pos_real:
+            reward += glMetric
+            appendRewardFeedback(
+                state,
+                f"Object '{obj_found}' found with correct position and stacking.",
+                glMetric,
+            )
+            objs.append(obj_found)
+        else:
+            mse = MSE(pos_real[:3], pos_found[:3])
+            reward -= mse
+            appendRewardFeedback(
+                state,
+                f"Object '{obj_found}' found with incorrect position. MSE: {mse:.2f}",
+                -mse,
+            )
+
+        if pos_found[3] != pos_real[3]:
+            reward -= glMetric / 4.0
+            appendRewardFeedback(
+                state,
+                f"Object '{obj_found}' found with incorrect stacking. Penalty: {glMetric / 4.0}",
+                -glMetric / 4.0,
+            )
+        else:
+            reward += glMetric / 4.0
+            appendRewardFeedback(
+                state,
+                f"Object '{obj_found}' found with correct stacking. Bonus: {glMetric / 4.0}",
+                glMetric / 4.0,
+            )
+
+    for obj in objs:
+        state.objectsLeft.remove(obj)
+        state.objectsFound.append(obj)
+
+    return reward