Fix health check timeout: start UI server in background before training

entrypoint.sh

@@ -21,7 +21,23 @@ if [ "$MODE" = "training" ]; then
     echo "========================================"
     echo "  OpenGrid — GRPO Training Mode"
     echo "========================================"
-    exec python run_training.py
+
+    # Start the UI server in background IMMEDIATELY so HF health check passes.
+    # Training output is written to training/outputs/ and the UI will serve it
+    # once training completes. The server stays alive throughout training.
+    echo "Starting background UI server on port 7860 (health check)..."
+    uvicorn app:app --host 0.0.0.0 --port 7860 &
+    UI_PID=$!
+
+    # Give server a moment to bind the port before training grabs GPU memory
+    sleep 5
+
+    # Run training (foreground)
+    python run_training.py
+
+    # Training finished — server is already running, just wait for it
+    echo "Training complete. UI server (PID $UI_PID) continues serving results."
+    wait $UI_PID
 else
     echo "========================================"
     echo "  OpenGrid — Control Room Server"

run_training.py

@@ -414,7 +414,13 @@ if __name__ == "__main__":
 
     # Start the full UI server (not a mini results server)
     # This serves the control room + training results on port 7860
-    print("\nTraining done. Starting full UI server on port 7860...")
-    import uvicorn
-    from app import app
-    uvicorn.run(app, host="0.0.0.0", port=7860)
+    # NOTE: In training mode, entrypoint.sh starts the server in background
+    # before training. This block is kept for standalone execution only.
+    if os.environ.get("OPENGRID_MODE") != "training":
+        print("\nTraining done. Starting full UI server on port 7860...")
+        import uvicorn
+        from app import app
+        uvicorn.run(app, host="0.0.0.0", port=7860)
+    else:
+        print("\nTraining done. UI server already running in background.")
+