feat: full clearning the repo

Dockerfile

@@ -18,5 +18,5 @@ COPY . .
 # As HF want for backend.
 EXPOSE 7860
 
-# Starting the server.  0.0.0.0 accepts connections from anywhere, not just local. 
+# 0.0.0.0 accepts connections from anywhere, not just local. also don't forget --ws to enable sockets on docker
 CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860", "--ws", "websockets"]

README.md

@@ -5,4 +5,6 @@ app_port: 7860
 pinned: false
 ---
 
-This project is part of a big project proiding Real-Time Tracking system for Indoors. 
+Tracking System Backend Deployed on huggingface.
+
+This repository is being modified only from the main Github Repo using CI/CD, Making sure it is always up to date and match project requirements.

api/routers/camera_stream.py

@@ -30,7 +30,7 @@ async def websocket_detect(
     
      url here is:  ws://127.0.0.1:8000/detectors/stream/camera_id
     """
-    # Yes, I asked the same questions, is using webscoket.app.state many times here is consuming.   after checking, it is not performance consuming. 
+    # Yes, I asked the same questions, is using webscoket.app.state many times here is consuming. after checking, it is not performance consuming. 
     state = websocket.app.state
     logger = state.logger
     # Using Depends is important and called Inversion Of Control (IoC)/ Dependency injection, and is important for testing.
@@ -72,37 +72,26 @@ async def websocket_detect(
     
     async def process_frames():
 
-
         try:
-            # What are the info you aim to collect from the camera? 
-            # How many frames received by second. 
-            # Frame processing time. 
-            # Average processing time  for logger. 
-            # Model processing time. 
-
-            # frame_count = itertools.count()
 
             logger.info(f"Camera {camera_id} start sending frames...")
 
-            def decode_frame():
-                # Decode image
-                return cv.imdecode(np.frombuffer(frame_bytes, np.uint8), cv.IMREAD_COLOR)        
+            def decode_frame(fb): return cv.imdecode(np.frombuffer(fb, np.uint8), cv.IMREAD_COLOR)        
 
             # Keep receiving messages in a loop until disconnection. 
             while True:
-
                 frame_bytes = await frame_queue.get()
                 
                 # Profiling
                 t0 = time.time()            
 
-                image_array = await loop.run_in_executor(None, decode_frame)
+                image_array = await loop.run_in_executor(None, decode_frame, frame_bytes)
                 decode_duration_seconds.labels(camera_id).observe(round(time.time() - t0, 3))
                 mlflow.log_metric("frame_processing_time", round(time.time() - t0, 3), next(step_counter))
 
+                # Apply detection models
                 detection_task = loop.run_in_executor(None, detector.detect, image_array)
                 safety_task = loop.run_in_executor(None, safety_detector.detect, image_array)
-
                 detections, safety_detection = await asyncio.gather(detection_task, safety_task)
                 detection_duration_seconds.labels(camera_id).observe(round(time.time() - t0, 3))
                 mlflow.log_metric("detection_duration_seconds", round(time.time() - t0, 3), next(step_counter))
@@ -129,7 +118,6 @@ async def websocket_detect(
                 detection_metadata = [DetectionMetadata(depth=depth, xRatio=xRatio) for depth, xRatio in zip(depth_points, boxes_center_ratio)]
                 metadata = CameraMetadata(camera_id=camera_id, is_danger = True if safety_detection else False, detection_metadata=detection_metadata)
                 
-                # state.camera_metadata[camera_id] = metadata.model_dump()
                 await redis.publish("dashboard_stream", metadata.model_dump_json())
                 # Even if the camera was disconnected, redis is still going to show its data, which is not accurate.
                 # Instead, we set expiry date for the camera data.
@@ -154,7 +142,6 @@ async def websocket_detect(
 
     except WebSocketDisconnect:
         logger.warn(f"Client ID >>{camera_id}<< Disconnected Normally...")
-        # state.camera_metadata.pop(camera_id, None)
 
     except Exception as e:
         logger.error(f"Error in websocker, Client ID: >>{camera_id}<<: {e}")
@@ -164,13 +151,4 @@ async def websocket_detect(
 
     finally:
         active_cameras.dec()
-        mlflow.end_run()
-
-
-# Uncomment this when needed, It is the same but using HTTP, which is Request Response only. could be used for testing. 
-# from fastapi import Request, UploadFile
-# @router.post("/detect")
-# async def post_detection(request: Request, file: UploadFile):
-#     # Request here is being used to access the app.state.model
-
-#     request.app.state.model.detect(file)
+        mlflow.end_run()

config/config.yaml

@@ -1,27 +1,17 @@
-# Shared defaults values that work for most people or envs
-# TODO, mostly this might not work in docker, due to the "\", feel free to replace it by [*proj_dir, ai, dl_models] for exmple
+# YAML file contains Shared defaults values that work for most people or envs
 
 project_name: Tracking Config
 # This one called Literal block, there is another one called Folded (>)
 project_desc: |
   Tracking System for detection, providing real-time camera status throught a Dashboard.
 task: indoor
-
 intervals:
-  system_metrics_seconds: 3.0  # Logging system metrics every
-  frames_summary_every: 30     # Number of frames to create a logs summary
-  realtime_updates_every: 2   # backend send updates every n seconds?
-
-prometheus_port: 9091
-
-paths:
-  # project_dir: &proj_dir G:\MyComputer\Workspace\Projects\gp-tracking-dashboard\tracking_dashboard
-  project_dir: &proj_dir .
-  # models_dir: &models_dir !join [*proj_dir, ai, dl_models]
-  # logs_dir:  !join [*proj_dir, backend, config, logs, logs] 
+  system_metrics_seconds: 3.0 # Logging system metrics every
+  frames_summary_every: 30 # Number of frames to create a logs summary
+  realtime_updates_every: 2 # backend send updates every n seconds?
 
 yolo:
-  # model_path: !join [*models_dir, yolo26s.pt] 
+  model_name: "yolo26n.pt"
   classes:
     - person
   batch_size: 16
@@ -31,11 +21,12 @@ yolo:
   data_path: ""
 
 security_detector:
-  # model_path: !join [*models_dir, "yolo_smoke_fire.pt"]
+  model_name: "yolo_smoke_fire.pt"
   classes:
     - fire
     - smoke
 
-depth: 
-  # model_path: !join [*models_dir, depth_anything_v2_vits.pth]  
-  encoder: "vits"
+depth:
+  model_name: "depth_anything_v2_vits.pth"
+  device: "cpu"
+  encoder: "vits"

config/settings.py

@@ -11,19 +11,18 @@ def join_tag(loader, node):
     parts = loader.construct_sequence(node)
     path = Path(*(str(part) for part in parts)).resolve()
     return str(path)
+
 # It didn't work before, After some research, .SafeLoaded is unmentioned must for my case. 
 yaml.SafeLoader.add_constructor("!join", join_tag)
 
-
-class PathsConfig(BaseModel):
-    """Contains paths of directories"""
-    project_dir: str
-    # models_dir: str
-    # logs_dir: str
+class IntervalsConfig(BaseModel):
+    system_metrics_seconds: float
+    frames_summary_every: int
+    realtime_updates_every: float
 
 class YoloConfig(BaseModel):
     """Contains yolo configurations"""
-    # model_path: str
+    model_name: str
     classes: List[str]
     batch_size: int
     epochs: int
@@ -33,19 +32,15 @@ class YoloConfig(BaseModel):
 
 class SecurityDetector(BaseModel):
     "Contains Security Detectors like Smoke - Fire"
-    # model_path: str
+    model_name: str
     classes: List[str]
 
 class DepthConfig(BaseModel):
     "Contains depths estimation configurations"
-    # model_path: str
+    model_name: str
+    device: Literal["cuda", "cpu"]
     encoder: Literal["vits", "vitb", "vitl", "vitg"]
 
-class IntervalsConfig(BaseModel):
-    system_metrics_seconds: float
-    frames_summary_every: int
-    realtime_updates_every: float
-
 
 class AppConfig(BaseSettings):
     """
@@ -59,33 +54,22 @@ class AppConfig(BaseSettings):
         env_file=Path(__file__).parent / ".env",
         env_file_encoding="utf-8",
         yaml_file=Path(__file__).parent / "config.yaml",
-        # case_sensitive=False,         # default True
-        # env_prefix="YOLO_",         # Means configs we are talking about starts with YOLO_
-        # env_nested_delimiter="__",  # Means we use _ instead of spaces for the same var
-        extra="ignore"              # Ignore other settings in yaml and env as they are not mentioedhere
+        extra="ignore"   # Ignore other settings in yaml and env as they are not mentioedhere
     )
 
     project_name:str
     project_desc:str
     task: Literal["indoor", "outdoor"]
 
-    paths: PathsConfig
     yolo: YoloConfig
     security_detector: SecurityDetector
     depth: DepthConfig
     intervals: IntervalsConfig
     redis_url:str
 
-    # Backend
-
-
     @classmethod
     def settings_customise_sources(cls, 
         settings_cls: type[BaseSettings],  # Base param.
-        # init_settings: PydanticBaseSettingsSource,   # Values passed to __init__
-        # env_settings: PydanticBaseSettingsSource,    # OS Env variables
-        # dotenv_settings: PydanticBaseSettingsSource, 
-        # file_secret_settings: PydanticBaseSettingsSource # Secret Directories
         **kwargs
         ) -> tuple[PydanticBaseSettingsSource, ...] :
         """
@@ -93,7 +77,7 @@ class AppConfig(BaseSettings):
         But this time it fixs the priority part, order by parameters priority.  
         """
 
-        # Order by priority
+        # Order by priority (first, more important)
         return (
             DotEnvSettingsSource(settings_cls),    # Most important
             EnvSettingsSource(settings_cls),       # This allow for ex. hugging face to override .env values with its values. 
@@ -101,43 +85,8 @@ class AppConfig(BaseSettings):
             )  # The return must be a tuple
 
 
-    # @classmethod
-    # def load_config(cls, yaml_path: Path | str = Path(__file__).parent / "config.yaml") -> "AppConfig":
-    #     """Loading confiuration and settings from Config.yaml file then override using .env"""
-
-    #     yaml_path = Path(yaml_path).resolve()  # Absolute path
-    #     if not yaml_path.is_file():
-    #         raise FileNotFoundError(f"Config file not found: {yaml_path}")
-
-    #     with yaml_path.open("r") as f:
-    #         yaml_data = yaml.safe_load(f) or {}
-
-    #     # env_data = cls() # This one loaded .env and not Yaml
-    #     # When this project grow, you are going to create different types of .yaml files for products and debugging and so on
-    #     # Feel free to stack them here, so we use the yaml required for our testing
-    #     # Note that in debuging.yaml file, we only override the base, not starting from scratch. 
-    #     # return cls(**{
-    #     #     **yaml_data,   # Loading config.yaml configurations
-    #     #     # **env_data.model_dump()     # TODO(FIX) Overriding everything using .env
-    #     #     })
-    #     return cls(**yaml_data)
-
-
-
-
 if __name__ == "__main__":  
-    # Note that we must use AppConfig without () to take .env in mind.
-    # Checking for YAML part.
-    # config = AppConfig.load_config()
-    # print(config.model_dump())
-    # print(config.model_dump()["project_name"])
-
-    # Checking for .env file. 
-    # config = AppConfig()
-    # print(f".env Path we are talking about: {Path(__file__).parent / ".env"}")
-    # print(config.model_config)
-    # print(config.project_name)
-
+    
     # Trying to checking both yaml and .env.   This works really fine now. 
     config = AppConfig()
     print(config.model_dump())

main.py

@@ -1,5 +1,4 @@
-from fastapi import FastAPI
-# from prometheus_client import metrics
+from ai.utils.hugging_face import hf_fetch_model
 from ai.depth.depth_anything import DepthAnything
 from ai.detectors.yolo_detector import YOLO_Detector
 from config.settings import AppConfig
@@ -8,15 +7,16 @@ from infra.system_metrics import log_system_metrics
 from api.routers import camera_stream
 from api.routers import dashboard_stream
 from api.routers import health
-from contextlib import asynccontextmanager
 from infra.logger_structlog import StructLogger
-import asyncio
+
+from contextlib import asynccontextmanager
 import mlflow
 import torch
-from huggingface_hub import hf_hub_download
 import redis.asyncio as aioredis
 from fastapi.middleware.trustedhost import TrustedHostMiddleware
 import dagshub
+from fastapi import FastAPI
+
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
@@ -26,28 +26,25 @@ async def lifespan(app: FastAPI):
     
     settings = AppConfig()
     logger = StructLogger(settings=settings)
+    # Using this way to can store data. it is acts as a dict which holds instances
+    app.state.logger = logger
+    app.state.settings = settings
    
     logger.info("Starting Server.... ")
-    asyncio.create_task(log_system_metrics(logger, logger_interval_sec=settings.intervals.system_metrics_seconds))    
+    # asyncio.create_task(log_system_metrics(logger, logger_interval_sec=settings.intervals.system_metrics_seconds))    
     
-    # Using this way to can store data. it is acts as a dict which holds instances
-    detection_model_path = hf_hub_download(repo_id="Ultralytics/YOLO26", filename="yolo26n.pt")
+    detection_model_path = hf_fetch_model(repo_id="Ultralytics/YOLO26", filename=settings.yolo.model_name)
     app.state.detection_model = YOLO_Detector(detection_model_path)
 
-    depth_model_path = hf_hub_download(repo_id="depth-anything/Depth-Anything-V2-Small", filename="depth_anything_v2_vits.pth")
-    app.state.depth_model = DepthAnything(encoder=settings.depth.encoder, depth_model_path=depth_model_path, DEVICE="cpu")
+    depth_model_path = hf_fetch_model(repo_id="depth-anything/Depth-Anything-V2-Small", filename=settings.depth.model_name)
+    app.state.depth_model = DepthAnything(encoder=settings.depth.encoder, depth_model_path=depth_model_path, DEVICE=settings.depth.device)
 
-    safety_detection_path = hf_hub_download(repo_id="e1250/safety_detection", filename="yolo_smoke_fire.pt")
+    safety_detection_path = hf_fetch_model(repo_id="e1250/safety_detection", filename=settings.security_detector.model_name)
     app.state.safety_detection_model = YOLO_Detector(safety_detection_path)
 
-    app.state.logger = logger
-    app.state.settings = settings
-    # app.state.camera_metadata = {}
-    # app.state.dashboard_clients = set()
-    # Redis(host="localhost", port=6379, db=0, decode_responses=True)
+
     app.state.redis = aioredis.from_url(settings.redis_url, decode_responses=True)
-    # Cnecking connection to redis. 
-    # Thinking of moving this to the health check. 
+    # Checking connection to redis - TODO add to health check 
     try:
         await app.state.redis.ping()
         logger.info("Redis connected successfully...")
@@ -55,15 +52,14 @@ async def lifespan(app: FastAPI):
         logger.error(f"Failed to connect to Redis: {e}")
         raise e
 
-    # Each camera should have its tracker to be able to work fine. 
-    # app.state.camera_trackers = {}
     yield
 
+    # Here You remove connections and release gpu here..
     logger.warn("Shutting down the server....")
     torch.cuda.empty_cache()
     await app.state.redis.close()
-    # You can remove connections and release gpu here .  
 
+# MLFlow setup
 dagshub.init(repo_owner='eslam760000', repo_name='p-tracking_system', mlflow=True)
 mlflow.set_tracking_uri("sqlite:///config/logs/mlflow.db")
 mlflow.set_experiment("realtime-detection-system")
@@ -76,7 +72,6 @@ app = FastAPI(
     lifespan=lifespan
     )
 
-# Allow
 app.add_middleware(
     TrustedHostMiddleware,
     allowed_hosts=["*"]

requirements.txt

@@ -1,22 +1,27 @@
+# Web
 fastapi
 uvicorn
 gunicorn
-websockets  
+websockets
 
+# AI
 ultralytics
+tracking_system@git+https://github.com/E1250/p-tracking_system.git@main
 
+# Arch
 pydantic
 pydantic_settings
 structlog
 
-tracking_system@git+https://github.com/E1250/p-tracking_system.git@main
 
 redis
 upstash_redis
+
+# Logging & Monitoring
 prometheus_client
+dagshub
+mlflow
 
 numpy 
 pandas
-mlflow
-huggingface_hub 
-dagshub
+huggingface_hub 

utils/experiment.py

@@ -1,12 +1,11 @@
+from backend.config.settings import AppConfig
 import mlflow
 
-def start_run(camera_id: str):
-    return mlflow.start_run(run_name=f"camera_{camera_id}")
-
+config = AppConfig()
 def log_config():
-    mlflow.log_param("detector", "yolov26_n")
-    mlflow.log_param("safety_model", "custom YOLO26_n")
-    mlflow.log_param("depth_model", "depthAnything_n")
+    mlflow.log_param("Detector", config.yolo.model_name)
+    mlflow.log_param("Safety Model", config.security_detector.model_name)
+    mlflow.log_param("Depth Model", config.depth.model_name)
 
 def log_metrics(metrics:dict):
     for k, v in metrics.items():