fix: add gateway watchdog + stale PID cleanup + retry logic

- Clean stale gateway.pid/lock files on startup (prevents 'already running' false positive)
- Retry gateway start up to 3 times with diagnostic log output on failure
- Extend startup timeout from 60s to 90s
- Add background watchdog that auto-restarts gateway every 30s if health check fails
- Watchdog cleans stale state before each restart attempt

start.sh

@@ -20,6 +20,13 @@ done
 
 echo "Persistent storage ready."
 
+# ── Clean up stale PID/lock files from previous crash ──
+echo "Cleaning up stale state..."
+rm -f "$HERMES_HOME/gateway.pid" 2>/dev/null
+rm -f "$HERMES_HOME/.gateway_runtime_lock" 2>/dev/null
+rm -f "$HERMES_HOME/.gateway_takeover" 2>/dev/null
+echo "Stale state cleaned."
+
 # Initialize MemPalace if not already
 PALACE_PATH="${MEMPALACE_PALACE_PATH:-/data/hermes/palace}"
 if [ ! -f "$PALACE_PATH/.palace_initialized" ]; then
@@ -31,25 +38,66 @@ else
     echo "MemPalace already initialized."
 fi
 
+# ── Gateway management functions ──
+start_gateway() {
+    # Clean stale PID/lock before each start attempt
+    rm -f "$HERMES_HOME/gateway.pid" 2>/dev/null
+    rm -f "$HERMES_HOME/.gateway_runtime_lock" 2>/dev/null
+    rm -f "$HERMES_HOME/.gateway_takeover" 2>/dev/null
+
+    echo "[$(date)] Starting Hermes Gateway..."
+    PYTHONUNBUFFERED=1 HERMES_ACCEPT_HOOKS=1 python3 -u -m hermes_cli.main gateway run -v \
+        >> /data/hermes/logs/gateway.log 2>&1 &
+    echo "$!" > /tmp/hermes-gateway.pid
+    echo "[$(date)] Gateway PID: $(cat /tmp/hermes-gateway.pid)"
+}
+
+wait_for_gateway() {
+    echo "[$(date)] Waiting for Gateway to start..."
+    for i in $(seq 1 45); do
+        if curl -s http://127.0.0.1:8642/health > /dev/null 2>&1; then
+            echo "[$(date)] Gateway is ready on :8642"
+            return 0
+        fi
+        # Check if process is still alive
+        if [ -f /tmp/hermes-gateway.pid ]; then
+            if ! kill -0 "$(cat /tmp/hermes-gateway.pid)" 2>/dev/null; then
+                echo "[$(date)] Gateway process died during startup (attempt $i/45)"
+                echo "[$(date)] --- Last 80 lines of gateway.log ---"
+                tail -80 /data/hermes/logs/gateway.log
+                echo "[$(date)] --- End gateway.log ---"
+                return 1
+            fi
+        fi
+        sleep 2
+    done
+    echo "[$(date)] Gateway startup timed out after 90s"
+    return 1
+}
+
 # Start Hermes Gateway (aiohttp API server on :8642 + Feishu platform)
-echo "Starting Hermes Gateway..."
-PYTHONUNBUFFERED=1 HERMES_ACCEPT_HOOKS=1 python3 -u -m hermes_cli.main gateway run -v \
-    >> /data/hermes/logs/gateway.log 2>&1 &
-GATEWAY_PID=$!
-echo "Gateway PID: $GATEWAY_PID"
-
-# Wait for gateway to be ready
-echo "Waiting for Gateway to start..."
-for i in $(seq 1 30); do
-    if curl -s http://127.0.0.1:8642/health > /dev/null 2>&1; then
-        echo "Gateway is ready on :8642"
+start_gateway
+
+# Wait for gateway to be ready, with retries
+GATEWAY_OK=false
+for attempt in 1 2 3; do
+    if wait_for_gateway; then
+        GATEWAY_OK=true
         break
     fi
-    sleep 2
+    if [ $attempt -lt 3 ]; then
+        echo "[$(date)] Retrying gateway start (attempt $((attempt+1))/3)..."
+        sleep 5
+        start_gateway
+    fi
 done
 
+if [ "$GATEWAY_OK" = false ]; then
+    echo "[$(date)] WARNING: Gateway failed after 3 attempts. Watchdog will keep retrying."
+fi
+
 # Start hermes-web-ui Node.js BFF server on :6060
-echo "Starting hermes-web-ui BFF..."
+echo "[$(date)] Starting hermes-web-ui BFF..."
 export PORT=6060
 export UPSTREAM=http://127.0.0.1:8642
 export HERMES_HOME=/root/.hermes
@@ -59,13 +107,13 @@ export NODE_ENV=production
 cd /app/webui-server
 node index.js >> /data/hermes/logs/webui.log 2>&1 &
 WEBUI_PID=$!
-echo "WebUI BFF PID: $WEBUI_PID"
+echo "[$(date)] WebUI BFF PID: $WEBUI_PID"
 
 # Wait for WebUI BFF to be ready
-echo "Waiting for WebUI BFF to start..."
+echo "[$(date)] Waiting for WebUI BFF to start..."
 for i in $(seq 1 30); do
     if curl -s http://127.0.0.1:6060/health > /dev/null 2>&1; then
-        echo "WebUI BFF is ready on :6060"
+        echo "[$(date)] WebUI BFF is ready on :6060"
         break
     fi
     sleep 2
@@ -79,5 +127,39 @@ echo "  Proxy:    http://0.0.0.0:7860"
 echo "  Auth Token: $AUTH_TOKEN"
 echo ""
 
+# ── Start Gateway watchdog in background ──
+(
+    echo "[$(date)] Gateway watchdog started (check interval: 30s)"
+    while true; do
+        sleep 30
+        # Check if gateway health endpoint responds
+        if ! curl -s --max-time 5 http://127.0.0.1:8642/health > /dev/null 2>&1; then
+            echo "[$(date)] Gateway health check FAILED — restarting..."
+            # Kill stale process if any
+            if [ -f /tmp/hermes-gateway.pid ]; then
+                OLD_PID=$(cat /tmp/hermes-gateway.pid)
+                kill -9 "$OLD_PID" 2>/dev/null || true
+            fi
+            # Append diagnostic info to log
+            echo "--- Gateway watchdog restart $(date) ---" >> /data/hermes/logs/gateway.log
+            # Clean stale state
+            rm -f "$HERMES_HOME/gateway.pid" 2>/dev/null
+            rm -f "$HERMES_HOME/.gateway_runtime_lock" 2>/dev/null
+            rm -f "$HERMES_HOME/.gateway_takeover" 2>/dev/null
+            # Restart
+            start_gateway
+            # Brief wait then verify
+            sleep 15
+            if curl -s --max-time 5 http://127.0.0.1:8642/health > /dev/null 2>&1; then
+                echo "[$(date)] Gateway restarted successfully"
+            else
+                echo "[$(date)] Gateway restart failed — will retry in 30s"
+            fi
+        fi
+    done
+) &
+WATCHDOG_PID=$!
+echo "[$(date)] Watchdog PID: $WATCHDOG_PID"
+
 # Start Python proxy on :7860 (main HF Space port)
 exec python3 /app/entry.py