fix: add Python-based gateway watchdog inside entry.py

Root cause: Gateway received SIGTERM twice in 30s. The shell-level
watchdog in start.sh also died because 'set -e' caused cascading
process termination. Gateway was left orphaned with no restart.

Fix 1: Python watchdog thread inside entry.py (daemon thread)
- Monitors gateway process every 30s using psutil
- Auto-restarts with --replace flag if gateway dies
- Uses start_new_session=True to decouple from entry.py signals
- Exponential backoff (30s → 60s → 120s → ... max 5min)
- Falls back to process name search if PID files are missing
- Survives shell-level SIGTERM because it's inside entry.py

Fix 2: Remove 'set -e' from start.sh
- Gateway restarts should NOT kill the entire script
- Shell-level startup is best-effort only
- Python watchdog in entry.py handles all restart logic

entry.py

@@ -624,6 +624,124 @@ class ProxyHandler(BaseHTTPRequestHandler):
             return []
 
 
+# ---------------------------------------------------------------------------
+# Python-based Gateway Watchdog (robust, lives inside entry.py)
+# ---------------------------------------------------------------------------
+
+def _gateway_watchdog(interval: int = 30):
+    """Monitor gateway process liveness and auto-restart if it dies.
+
+    This runs as a daemon thread inside entry.py so it survives
+    SIGTERM to the shell-level watchdog in start.sh.
+    """
+    import signal as _signal
+    pid_file = os.path.join(HERMES_HOME, "gateway.pid")
+    alt_pid_file = "/tmp/hermes-gateway.pid"
+    lock_file = os.path.join(HERMES_HOME, ".gateway_runtime_lock")
+    takeover_file = os.path.join(HERMES_HOME, ".gateway_takeover")
+    gw_log = "/data/hermes/logs/gateway.log"
+
+    def _find_gateway_pid() -> int | None:
+        """Find the gateway PID from pid files or by process name."""
+        for pf in (pid_file, alt_pid_file):
+            try:
+                with open(pf) as f:
+                    pid = int(f.read().strip())
+                if pid > 0 and psutil.pid_exists(pid):
+                    proc = psutil.Process(pid)
+                    if "hermes" in " ".join(proc.cmdline()).lower() or "gateway" in " ".join(proc.cmdline()).lower():
+                        return pid
+            except Exception:
+                pass
+        # Fallback: search by process name
+        try:
+            for proc in psutil.process_iter(["pid", "cmdline"]):
+                try:
+                    cmdline = " ".join(proc.info["cmdline"] or [])
+                    if "hermes_cli.main" in cmdline and "gateway" in cmdline:
+                        return proc.info["pid"]
+                except (psutil.NoSuchProcess, psutil.AccessDenied):
+                    pass
+        except Exception:
+            pass
+        return None
+
+    def _start_gateway():
+        """Start the gateway process."""
+        # Clean stale state
+        for f in (pid_file, lock_file, takeover_file):
+            try:
+                os.remove(f)
+            except Exception:
+                pass
+
+        log_fh = None
+        try:
+            log_fh = open(gw_log, "a")
+            log_fh.write(f"\n--- [Python Watchdog] Starting gateway at {datetime.now().isoformat()} ---\n")
+            log_fh.flush()
+        except Exception:
+            pass
+
+        env = os.environ.copy()
+        env["PYTHONUNBUFFERED"] = "1"
+        env["HERMES_ACCEPT_HOOKS"] = "1"
+        proc = subprocess.Popen(
+            [sys.executable, "-u", "-m", "hermes_cli.main", "gateway", "run", "-v", "--replace"],
+            stdout=log_fh if log_fh else subprocess.DEVNULL,
+            stderr=subprocess.STDOUT,
+            env=env,
+            start_new_session=True,  # decouple from entry.py signals
+        )
+        try:
+            with open(alt_pid_file, "w") as f:
+                f.write(str(proc.pid))
+        except Exception:
+            pass
+        return proc
+
+    logger.info("[Watchdog] Python gateway watchdog started (interval=%ds)", interval)
+    time.sleep(10)  # Let initial startup settle
+
+    last_restart = 0
+    restart_backoff = 30
+
+    while True:
+        try:
+            time.sleep(interval)
+
+            gw_pid = _find_gateway_pid()
+            if gw_pid is not None:
+                # Gateway is alive, reset backoff
+                restart_backoff = 30
+                continue
+
+            # Gateway is dead — restart with backoff
+            now = time.time()
+            if now - last_restart < restart_backoff:
+                continue
+
+            logger.warning("[Watchdog] Gateway process is DEAD — restarting...")
+            proc = _start_gateway()
+            last_restart = now
+            restart_backoff = min(restart_backoff * 2, 300)  # exponential backoff, max 5min
+
+            # Wait and verify
+            time.sleep(15)
+            try:
+                if proc.poll() is None:
+                    logger.info("[Watchdog] Gateway restarted successfully (PID=%d)", proc.pid)
+                    restart_backoff = 30  # reset on success
+                else:
+                    logger.error("[Watchdog] Gateway exited immediately with code %d", proc.returncode)
+            except Exception as e:
+                logger.error("[Watchdog] Failed to verify restart: %s", e)
+
+        except Exception as e:
+            logger.error("[Watchdog] Error: %s", e)
+            time.sleep(10)
+
+
 # ---------------------------------------------------------------------------
 # Main
 # ---------------------------------------------------------------------------
@@ -642,6 +760,11 @@ def main():
     tailer.start()
     logger.info("Log tailer started")
 
+    # Start Python-based gateway watchdog (survives shell death)
+    watchdog = threading.Thread(target=_gateway_watchdog, args=(30,), daemon=True, name="gateway-watchdog")
+    watchdog.start()
+    logger.info("Python gateway watchdog started")
+
     # Check if backend services are reachable
     for attempt in range(10):
         try:

start.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
-set -e
+# Hermes Bot — HuggingFace Space Startup
+# NOTE: No 'set -e' — gateway restarts should not kill the entire script
 
 echo "=== Hermes Bot — HuggingFace Space Startup ==="
 
@@ -25,6 +26,7 @@ echo "Cleaning up stale state..."
 rm -f "$HERMES_HOME/gateway.pid" 2>/dev/null
 rm -f "$HERMES_HOME/.gateway_runtime_lock" 2>/dev/null
 rm -f "$HERMES_HOME/.gateway_takeover" 2>/dev/null
+rm -f /tmp/hermes-gateway.pid 2>/dev/null
 echo "Stale state cleaned."
 
 # Initialize MemPalace if not already
@@ -38,65 +40,32 @@ else
     echo "MemPalace already initialized."
 fi
 
-# ── Gateway management functions ──
-start_gateway() {
-    # Clean stale PID/lock before each start attempt
-    rm -f "$HERMES_HOME/gateway.pid" 2>/dev/null
-    rm -f "$HERMES_HOME/.gateway_runtime_lock" 2>/dev/null
-    rm -f "$HERMES_HOME/.gateway_takeover" 2>/dev/null
+# ── Start Hermes Gateway ──
+echo "[$(date)] Starting Hermes Gateway..."
+PYTHONUNBUFFERED=1 HERMES_ACCEPT_HOOKS=1 python3 -u -m hermes_cli.main gateway run -v \
+    >> /data/hermes/logs/gateway.log 2>&1 &
+GATEWAY_PID=$!
+echo "[$(date)] Gateway PID: $GATEWAY_PID"
+echo "$GATEWAY_PID" > /tmp/hermes-gateway.pid
 
-    echo "[$(date)] Starting Hermes Gateway..."
-    PYTHONUNBUFFERED=1 HERMES_ACCEPT_HOOKS=1 python3 -u -m hermes_cli.main gateway run -v \
-        >> /data/hermes/logs/gateway.log 2>&1 &
-    echo "$!" > /tmp/hermes-gateway.pid
-    echo "[$(date)] Gateway PID: $(cat /tmp/hermes-gateway.pid)"
-}
-
-wait_for_gateway() {
-    echo "[$(date)] Waiting for Gateway to start..."
-    for i in $(seq 1 45); do
-        if curl -s http://127.0.0.1:8642/health > /dev/null 2>&1; then
-            echo "[$(date)] Gateway is ready on :8642"
-            return 0
-        fi
-        # Check if process is still alive
-        if [ -f /tmp/hermes-gateway.pid ]; then
-            if ! kill -0 "$(cat /tmp/hermes-gateway.pid)" 2>/dev/null; then
-                echo "[$(date)] Gateway process died during startup (attempt $i/45)"
-                echo "[$(date)] --- Last 80 lines of gateway.log ---"
-                tail -80 /data/hermes/logs/gateway.log
-                echo "[$(date)] --- End gateway.log ---"
-                return 1
-            fi
-        fi
-        sleep 2
-    done
-    echo "[$(date)] Gateway startup timed out after 90s"
-    return 1
-}
-
-# Start Hermes Gateway (aiohttp API server on :8642 + Feishu platform)
-start_gateway
-
-# Wait for gateway to be ready, with retries
-GATEWAY_OK=false
-for attempt in 1 2 3; do
-    if wait_for_gateway; then
-        GATEWAY_OK=true
+# Wait for gateway to be ready (best-effort, no set -e)
+echo "[$(date)] Waiting for Gateway to start..."
+for i in $(seq 1 30); do
+    if curl -s http://127.0.0.1:8642/health > /dev/null 2>&1; then
+        echo "[$(date)] Gateway is ready on :8642"
         break
     fi
-    if [ $attempt -lt 3 ]; then
-        echo "[$(date)] Retrying gateway start (attempt $((attempt+1))/3)..."
-        sleep 5
-        start_gateway
+    # Check if process is still alive
+    if ! kill -0 "$GATEWAY_PID" 2>/dev/null; then
+        echo "[$(date)] Gateway process died during startup"
+        tail -30 /data/hermes/logs/gateway.log 2>/dev/null
+        echo "[$(date)] NOTE: Python watchdog in entry.py will auto-restart gateway"
+        break
     fi
+    sleep 2
 done
 
-if [ "$GATEWAY_OK" = false ]; then
-    echo "[$(date)] WARNING: Gateway failed after 3 attempts. Watchdog will keep retrying."
-fi
-
-# Start hermes-web-ui Node.js BFF server on :6060
+# ── Start hermes-web-ui Node.js BFF server on :6060 ──
 echo "[$(date)] Starting hermes-web-ui BFF..."
 export PORT=6060
 export UPSTREAM=http://127.0.0.1:8642
@@ -111,7 +80,7 @@ echo "[$(date)] WebUI BFF PID: $WEBUI_PID"
 
 # Wait for WebUI BFF to be ready
 echo "[$(date)] Waiting for WebUI BFF to start..."
-for i in $(seq 1 30); do
+for i in $(seq 1 15); do
     if curl -s http://127.0.0.1:6060/health > /dev/null 2>&1; then
         echo "[$(date)] WebUI BFF is ready on :6060"
         break
@@ -121,52 +90,13 @@ done
 
 echo ""
 echo "=== All services started ==="
-echo "  Gateway:  http://127.0.0.1:8642"
+echo "  Gateway:  http://127.0.0.1:8642 (with Python watchdog in entry.py)"
 echo "  WebUI:    http://127.0.0.1:6060"
 echo "  Proxy:    http://0.0.0.0:7860"
 echo "  Auth Token: $AUTH_TOKEN"
 echo ""
 
-# ── Start Gateway watchdog in background ──
-(
-    echo "[$(date)] Gateway watchdog started (check interval: 60s)"
-    while true; do
-        sleep 60
-        # Check if gateway process is still alive
-        GW_ALIVE=false
-        if [ -f /tmp/hermes-gateway.pid ]; then
-            if kill -0 "$(cat /tmp/hermes-gateway.pid)" 2>/dev/null; then
-                GW_ALIVE=true
-            fi
-        fi
-        # Also check via gateway.pid in hermes home
-        if [ "$GW_ALIVE" = false ] && [ -f "$HERMES_HOME/gateway.pid" ]; then
-            if kill -0 "$(cat "$HERMES_HOME/gateway.pid")" 2>/dev/null; then
-                GW_ALIVE=true
-                # Sync the PID file
-                cp "$HERMES_HOME/gateway.pid" /tmp/hermes-gateway.pid
-            fi
-        fi
-
-        if [ "$GW_ALIVE" = false ]; then
-            echo "[$(date)] Gateway process is DEAD — restarting..."
-            # Clean stale state
-            rm -f "$HERMES_HOME/gateway.pid" 2>/dev/null
-            rm -f "$HERMES_HOME/.gateway_runtime_lock" 2>/dev/null
-            rm -f "$HERMES_HOME/.gateway_takeover" 2>/dev/null
-            # Restart
-            start_gateway
-            sleep 15
-            if [ -f /tmp/hermes-gateway.pid ] && kill -0 "$(cat /tmp/hermes-gateway.pid)" 2>/dev/null; then
-                echo "[$(date)] Gateway restarted successfully"
-            else
-                echo "[$(date)] Gateway restart failed — will retry in 60s"
-            fi
-        fi
-    done
-) &
-WATCHDOG_PID=$!
-echo "[$(date)] Watchdog PID: $WATCHDOG_PID"
-
 # Start Python proxy on :7860 (main HF Space port)
+# entry.py contains a Python-based gateway watchdog that will auto-restart
+# the gateway if it dies, regardless of what happens to this shell script
 exec python3 /app/entry.py