Merge pull request #15 from anurag162008/codex/investigate-changes-in-openclaw.json-backup

Improve workspace sync (config watch/settle, locking, restart sync) and key-rotator logging

openclaw-sync.py

@@ -7,6 +7,7 @@ credentials inside a private HF dataset without embedding HF tokens in git
 remotes or requiring a manual HF_USERNAME secret.
 """
 
+import fcntl
 import hashlib
 import json
 import logging
@@ -37,6 +38,7 @@ OPENCLAW_HOME = Path("/home/node/.openclaw")
 OPENCLAW_CONFIG_FILE = OPENCLAW_HOME / "openclaw.json"
 WORKSPACE = OPENCLAW_HOME / "workspace"
 STATUS_FILE = Path("/tmp/sync-status.json")
+SYNC_LOCK_FILE = Path("/tmp/huggingclaw-sync.lock")
 INTERVAL = int(os.environ.get("SYNC_INTERVAL", "180"))
 INITIAL_DELAY = int(os.environ.get("SYNC_START_DELAY", "10"))
 CONFIG_WATCH_INTERVAL = max(
@@ -392,7 +394,7 @@ def restore_workspace() -> bool:
         return False
 
 
-def sync_once(
+def _sync_once_unlocked(
     last_fingerprint: str | None = None,
     last_marker: tuple[int, int, int] | None = None,
 ) -> tuple[str, tuple[int, int, int]]:
@@ -440,6 +442,19 @@ def sync_once(
     return (current_fingerprint, current_marker)
 
 
+def sync_once(
+    last_fingerprint: str | None = None,
+    last_marker: tuple[int, int, int] | None = None,
+) -> tuple[str, tuple[int, int, int]]:
+    SYNC_LOCK_FILE.parent.mkdir(parents=True, exist_ok=True)
+    with SYNC_LOCK_FILE.open("w", encoding="utf-8") as lock_handle:
+        fcntl.flock(lock_handle, fcntl.LOCK_EX)
+        try:
+            return _sync_once_unlocked(last_fingerprint, last_marker)
+        finally:
+            fcntl.flock(lock_handle, fcntl.LOCK_UN)
+
+
 def handle_signal(_sig, _frame) -> None:
     STOP_EVENT.set()
 
@@ -583,6 +598,17 @@ def main() -> int:
             write_status("error", f"Shutdown sync failed: {exc}")
             print(f"Workspace sync: shutdown sync failed: {exc}")
             return 1
+    if command == "sync-once-settled":
+        try:
+            trigger, _ = wait_for_config_settle(file_marker(OPENCLAW_CONFIG_FILE))
+            if trigger == "stopped":
+                return 1
+            sync_once()
+            return 0
+        except Exception as exc:
+            write_status("error", f"Settled sync failed: {exc}")
+            print(f"Workspace sync: settled sync failed: {exc}")
+            return 1
     if command == "loop":
         return loop()
 

start.sh

@@ -1018,59 +1018,108 @@ hc_finish_startup_commands
 sync_installed_plugins_into_allow
 
 # ── Launch gateway ──
-echo "Launching OpenClaw gateway on port 7860..."
+GATEWAY_RESTART_DELAY="${GATEWAY_RESTART_DELAY:-2}"
+GATEWAY_MAX_RESTARTS="${GATEWAY_MAX_RESTARTS:-0}"
+GATEWAY_RESTART_COUNT=0
+SYNC_LOOP_PID=""
+GUARDIAN_PID=""
+
+sync_before_gateway_restart() {
+  [ -n "${HF_TOKEN:-}" ] || return 0
+  [ -f "/home/node/app/openclaw-sync.py" ] || return 0
+
+  echo "Gateway stopped; saving latest OpenClaw state before restart..."
+  python3 /home/node/app/openclaw-sync.py sync-once-settled || \
+    echo "Warning: could not sync settled state before gateway restart"
+}
 
-GATEWAY_ARGS=(gateway run --port 7860 --bind lan)
-if [ "${GATEWAY_VERBOSE:-0}" = "1" ]; then
-  GATEWAY_ARGS+=(--verbose)
-  echo "Gateway verbose logging enabled (GATEWAY_VERBOSE=1)"
-fi
+start_background_sync_once() {
+  [ -n "${HF_TOKEN:-}" ] || return 0
 
-# Use stdbuf -oL -eL to ensure logs are not buffered and appear immediately
-# in the console. NOTE: $! captures the LAST pipeline element (tee), not
-# openclaw — fine for passing to `wait` (waits for the whole pipeline to
-# finish), but kill -0 on it is uninformative. We probe TCP instead.
-stdbuf -oL -eL openclaw "${GATEWAY_ARGS[@]}" 2>&1 | tee -a /home/node/.openclaw/gateway.log &
-GATEWAY_PID=$!
-
-# Poll for the gateway to start listening on 7860. OpenClaw can take 20-30s
-# on cold start (plugin install + auto-restore). Bail out early if the
-# pipeline died.
-GATEWAY_READY_TIMEOUT="${GATEWAY_READY_TIMEOUT:-90}"
-ready=false
-for ((i=0; i<GATEWAY_READY_TIMEOUT; i++)); do
-  if (echo > /dev/tcp/127.0.0.1/7860) 2>/dev/null; then
-    ready=true
-    break
-  fi
-  if ! kill -0 "$GATEWAY_PID" 2>/dev/null; then
-    break
+  if [ -n "$SYNC_LOOP_PID" ] && kill -0 "$SYNC_LOOP_PID" 2>/dev/null; then
+    return 0
   fi
-  sleep 1
-done
 
-if [ "$ready" != "true" ]; then
-  echo ""
-  echo "Gateway failed to start. Last 30 lines of log:"
-  echo "────────────────────────────────────────────"
-  tail -30 /home/node/.openclaw/gateway.log
-  exit 1
-fi
+  python3 -u /home/node/app/openclaw-sync.py loop &
+  SYNC_LOOP_PID=$!
+}
+
+start_guardian_once() {
+  [ "$WHATSAPP_ENABLED_NORMALIZED" = "true" ] || return 0
+
+  if [ -n "$GUARDIAN_PID" ] && kill -0 "$GUARDIAN_PID" 2>/dev/null; then
+    return 0
+  fi
 
-# 11. Start WhatsApp Guardian after the gateway is accepting connections
-if [ "$WHATSAPP_ENABLED_NORMALIZED" = "true" ]; then
   node /home/node/app/wa-guardian.js &
   GUARDIAN_PID=$!
   echo "WhatsApp Guardian started (PID: $GUARDIAN_PID)"
-fi
+}
 
-# 11.5 Warm up the managed browser so first browser actions have a live tab
-warmup_browser
+while true; do
+  echo "Launching OpenClaw gateway on port 7860..."
 
-# 12. Start Workspace Sync after startup settles
-if [ -n "${HF_TOKEN:-}" ]; then
-  python3 -u /home/node/app/openclaw-sync.py loop &
-fi
+  GATEWAY_ARGS=(gateway run --port 7860 --bind lan)
+  if [ "${GATEWAY_VERBOSE:-0}" = "1" ]; then
+    GATEWAY_ARGS+=(--verbose)
+    echo "Gateway verbose logging enabled (GATEWAY_VERBOSE=1)"
+  fi
+
+  # Use stdbuf -oL -eL to ensure logs are not buffered and appear immediately
+  # in the console. NOTE: $! captures the LAST pipeline element (tee), not
+  # openclaw — fine for passing to `wait` (waits for the whole pipeline to
+  # finish), but kill -0 on it is uninformative. We probe TCP instead.
+  stdbuf -oL -eL openclaw "${GATEWAY_ARGS[@]}" 2>&1 | tee -a /home/node/.openclaw/gateway.log &
+  GATEWAY_PID=$!
+
+  # Poll for the gateway to start listening on 7860. OpenClaw can take 20-30s
+  # on cold start (plugin install + auto-restore). Bail out early if the
+  # pipeline died.
+  GATEWAY_READY_TIMEOUT="${GATEWAY_READY_TIMEOUT:-90}"
+  ready=false
+  for ((i=0; i<GATEWAY_READY_TIMEOUT; i++)); do
+    if (echo > /dev/tcp/127.0.0.1/7860) 2>/dev/null; then
+      ready=true
+      break
+    fi
+    if ! kill -0 "$GATEWAY_PID" 2>/dev/null; then
+      break
+    fi
+    sleep 1
+  done
+
+  if [ "$ready" != "true" ]; then
+    echo ""
+    echo "Gateway failed to start. Last 30 lines of log:"
+    echo "────────────────────────────────────────────"
+    tail -30 /home/node/.openclaw/gateway.log
+    exit 1
+  fi
+
+  # 11. Start WhatsApp Guardian after the gateway is accepting connections
+  start_guardian_once
 
-# Wait for gateway (allows trap to fire)
-wait $GATEWAY_PID
+  # 11.5 Warm up the managed browser so first browser actions have a live tab
+  warmup_browser
+
+  # 12. Start Workspace Sync after startup settles. Keep only one loop active;
+  # config edits can make OpenClaw exit/reload, and the gateway loop below will
+  # relaunch it without rerunning all startup code.
+  start_background_sync_once
+
+  set +e
+  wait "$GATEWAY_PID"
+  GATEWAY_EXIT_CODE=$?
+  set -e
+
+  sync_before_gateway_restart
+
+  GATEWAY_RESTART_COUNT=$((GATEWAY_RESTART_COUNT + 1))
+  if [ "$GATEWAY_MAX_RESTARTS" != "0" ] && [ "$GATEWAY_RESTART_COUNT" -ge "$GATEWAY_MAX_RESTARTS" ]; then
+    echo "Gateway exited with code ${GATEWAY_EXIT_CODE}; restart limit (${GATEWAY_MAX_RESTARTS}) reached."
+    exit "$GATEWAY_EXIT_CODE"
+  fi
+
+  echo "Gateway exited with code ${GATEWAY_EXIT_CODE}; restarting in ${GATEWAY_RESTART_DELAY}s..."
+  sleep "$GATEWAY_RESTART_DELAY"
+done