Sync state before gateway reload restarts

.env.example

@@ -285,6 +285,12 @@ KEEP_ALIVE_INTERVAL=300
 # Workspace auto-sync interval (seconds). Default: 180.
 SYNC_INTERVAL=180
 
+# Check openclaw.json for settings changes this often (seconds). Default: 1.
+OPENCLAW_CONFIG_WATCH_INTERVAL=1
+
+# Wait for openclaw.json to stay valid and unchanged before syncing. Default: 3.
+OPENCLAW_CONFIG_SETTLE_SECONDS=3
+
 # Webhooks: Standard POST notifications for lifecycle events
 # WEBHOOK_URL=https://your-webhook-endpoint.com/log
 

README.md

@@ -162,7 +162,9 @@ HuggingClaw automatically syncs your workspace (chats, settings, sessions) to a
 | Variable | Default | Description |
 | :--- | :--- | :--- |
 | `HF_TOKEN` | — | HF token with **Write** access |
-| `SYNC_INTERVAL` | `180` | Backup frequency in seconds |
+| `SYNC_INTERVAL` | `180` | Full backup frequency in seconds |
+| `OPENCLAW_CONFIG_WATCH_INTERVAL` | `1` | How often to check `openclaw.json` for immediate settings sync |
+| `OPENCLAW_CONFIG_SETTLE_SECONDS` | `3` | How long `openclaw.json` must stay valid and unchanged before syncing |
 
 ## 📦 Ephemeral Package Re-install *(Optional)*
 

multi-provider-key-rotator.cjs

@@ -19,6 +19,11 @@
 const http  = require('node:http');
 const https = require('node:https');
 
+// This file is preloaded through NODE_OPTIONS, so it also runs inside npm and
+// OpenClaw helper subprocesses that may emit machine-readable JSON on stdout.
+// Keep rotator diagnostics on stderr to avoid corrupting those stdout streams.
+const log = (...args) => console.error(...args);
+
 // ─── Provider definitions ────────────────────────────────────────────────────
 //
 // hostname  – regex tested against the request hostname (case-insensitive)
@@ -185,7 +190,7 @@ const providerState = PROVIDERS.map(p => {
     : normalizeKeys(process.env.LLM_API_KEY || '');
 
   if (hasDedicated) {
-    console.log(`[key-rotator] ${p.name}: ${keys.length} key${keys.length === 1 ? '' : 's'}`);
+    log(`[key-rotator] ${p.name}: ${keys.length} key${keys.length === 1 ? '' : 's'}`);
   } else if (!keys.length) {
     console.warn(`[key-rotator] No keys for provider "${p.name}"`);
   }
@@ -202,7 +207,7 @@ const fallbackCount = providerState.filter(p => {
   return dedicated.length === 0 && p.keys.length > 0;
 }).length;
 if (fallbackCount > 0) {
-  console.log(`[key-rotator] ${fallbackCount} provider(s) using LLM_API_KEY fallback`);
+  log(`[key-rotator] ${fallbackCount} provider(s) using LLM_API_KEY fallback`);
 }
 
 // ─── Runtime helpers ─────────────────────────────────────────────────────────
@@ -332,4 +337,4 @@ patchFetch();
 patchHttpModule(http);
 patchHttpModule(https);
 
-console.log('[key-rotator] loaded — all providers active');
+log('[key-rotator] loaded — all providers active');

openclaw-sync.py

@@ -7,6 +7,7 @@ credentials inside a private HF dataset without embedding HF tokens in git
 remotes or requiring a manual HF_USERNAME secret.
 """
 
+import fcntl
 import hashlib
 import json
 import logging
@@ -34,10 +35,20 @@ from huggingface_hub.errors import HfHubHTTPError, RepositoryNotFoundError
 logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
 
 OPENCLAW_HOME = Path("/home/node/.openclaw")
+OPENCLAW_CONFIG_FILE = OPENCLAW_HOME / "openclaw.json"
 WORKSPACE = OPENCLAW_HOME / "workspace"
 STATUS_FILE = Path("/tmp/sync-status.json")
+SYNC_LOCK_FILE = Path("/tmp/huggingclaw-sync.lock")
 INTERVAL = int(os.environ.get("SYNC_INTERVAL", "180"))
 INITIAL_DELAY = int(os.environ.get("SYNC_START_DELAY", "10"))
+CONFIG_WATCH_INTERVAL = max(
+    0.5,
+    float(os.environ.get("OPENCLAW_CONFIG_WATCH_INTERVAL", "1")),
+)
+CONFIG_SETTLE_SECONDS = max(
+    0.0,
+    float(os.environ.get("OPENCLAW_CONFIG_SETTLE_SECONDS", "3")),
+)
 HF_TOKEN = os.environ.get("HF_TOKEN", "").strip()
 HF_USERNAME = os.environ.get("HF_USERNAME", "").strip()
 SPACE_AUTHOR_NAME = os.environ.get("SPACE_AUTHOR_NAME", "").strip()
@@ -78,6 +89,13 @@ def write_status(status: str, message: str) -> None:
     tmp_path.replace(STATUS_FILE)
 
 
+def read_status() -> dict[str, str]:
+    try:
+        return json.loads(STATUS_FILE.read_text(encoding="utf-8"))
+    except Exception:
+        return {}
+
+
 def count_files(path: Path) -> int:
     if not path.exists():
         return 0
@@ -250,6 +268,18 @@ def _should_exclude(rel_posix: str, path: Path) -> bool:
     return False
 
 
+def file_marker(path: Path) -> tuple[int, int, int]:
+    try:
+        stat = path.stat()
+    except OSError:
+        return (0, 0, 0)
+
+    if not path.is_file():
+        return (0, 0, 0)
+
+    return (1, int(stat.st_size), int(stat.st_mtime_ns))
+
+
 def metadata_marker(root: Path) -> tuple[int, int, int]:
     if not root.exists():
         return (0, 0, 0)
@@ -364,7 +394,7 @@ def restore_workspace() -> bool:
         return False
 
 
-def sync_once(
+def _sync_once_unlocked(
     last_fingerprint: str | None = None,
     last_marker: tuple[int, int, int] | None = None,
 ) -> tuple[str, tuple[int, int, int]]:
@@ -412,14 +442,81 @@ def sync_once(
     return (current_fingerprint, current_marker)
 
 
+def sync_once(
+    last_fingerprint: str | None = None,
+    last_marker: tuple[int, int, int] | None = None,
+) -> tuple[str, tuple[int, int, int]]:
+    SYNC_LOCK_FILE.parent.mkdir(parents=True, exist_ok=True)
+    with SYNC_LOCK_FILE.open("w", encoding="utf-8") as lock_handle:
+        fcntl.flock(lock_handle, fcntl.LOCK_EX)
+        try:
+            return _sync_once_unlocked(last_fingerprint, last_marker)
+        finally:
+            fcntl.flock(lock_handle, fcntl.LOCK_UN)
+
+
 def handle_signal(_sig, _frame) -> None:
     STOP_EVENT.set()
 
 
+def is_valid_json_file(path: Path) -> bool:
+    if not path.exists():
+        return True
+
+    try:
+        json.loads(path.read_text(encoding="utf-8"))
+        return True
+    except Exception:
+        return False
+
+
+def wait_for_config_settle(config_marker: tuple[int, int, int]) -> tuple[str, tuple[int, int, int]]:
+    stable_since = time.monotonic()
+    current_marker = config_marker
+
+    while not STOP_EVENT.is_set():
+        latest_marker = file_marker(OPENCLAW_CONFIG_FILE)
+        if latest_marker != current_marker:
+            current_marker = latest_marker
+            stable_since = time.monotonic()
+
+        if (
+            time.monotonic() - stable_since >= CONFIG_SETTLE_SECONDS
+            and is_valid_json_file(OPENCLAW_CONFIG_FILE)
+        ):
+            return ("settled", current_marker)
+
+        if STOP_EVENT.wait(CONFIG_WATCH_INTERVAL):
+            return ("stopped", current_marker)
+
+    return ("stopped", current_marker)
+
+
+def wait_for_sync_trigger(config_marker: tuple[int, int, int]) -> tuple[str, tuple[int, int, int]]:
+    deadline = time.monotonic() + max(0, INTERVAL)
+
+    while not STOP_EVENT.is_set():
+        current_config_marker = file_marker(OPENCLAW_CONFIG_FILE)
+        if current_config_marker != config_marker:
+            return wait_for_config_settle(current_config_marker)
+
+        remaining = deadline - time.monotonic()
+        if remaining <= 0:
+            return ("interval", current_config_marker)
+
+        wait_seconds = min(CONFIG_WATCH_INTERVAL, remaining)
+        if STOP_EVENT.wait(wait_seconds):
+            return ("stopped", current_config_marker)
+
+    return ("stopped", config_marker)
+
+
 def loop() -> int:
     signal.signal(signal.SIGTERM, handle_signal)
     signal.signal(signal.SIGINT, handle_signal)
 
+    previous_status = read_status().get("status", "")
+
     try:
         repo_id = resolve_backup_namespace()
         write_status("configured", f"Backup loop active for {repo_id} with {INTERVAL}s interval.")
@@ -431,24 +528,56 @@ def loop() -> int:
     time.sleep(INITIAL_DELAY)
     print(f"Workspace sync started: every {INTERVAL}s -> {repo_id}")
 
-    # Take a fingerprint of the workspace AS RESTORED (after snapshotting state)
-    # so the first loop iteration only uploads if something genuinely changed.
-    # Previously this was None, which forced an unconditional upload every restart
-    # — even when restore had failed silently and the workspace was empty.
+    # Capture the restored dataset state before refreshing the embedded
+    # /home/node/.openclaw backup.  Startup may have patched openclaw.json
+    # after restore (token/model/logging/channel toggles), and that patch only
+    # becomes part of the dataset once snapshot_state_into_workspace() copies it
+    # into workspace/huggingclaw-state/openclaw/.  If the snapshot changes the
+    # workspace, seed the first sync with the pre-snapshot fingerprint so the
+    # updated openclaw.json is uploaded instead of being treated as the baseline.
+    pre_snapshot_fingerprint = fingerprint_dir(WORKSPACE)
+    pre_snapshot_marker = metadata_marker(WORKSPACE)
     snapshot_state_into_workspace()
     last_fingerprint = fingerprint_dir(WORKSPACE)
     last_marker = metadata_marker(WORKSPACE)
-    print("Initial workspace fingerprint captured.")
+
+    if last_fingerprint != pre_snapshot_fingerprint:
+        if previous_status == "error":
+            print(
+                "Initial state snapshot changed, but restore previously failed; "
+                "keeping current state as baseline to avoid overwriting the remote backup."
+            )
+        else:
+            last_fingerprint = pre_snapshot_fingerprint
+            last_marker = pre_snapshot_marker
+            print("Initial state snapshot changed; first sync will upload refreshed OpenClaw state.")
+    else:
+        print("Initial workspace fingerprint captured.")
+
+    config_marker = file_marker(OPENCLAW_CONFIG_FILE)
 
     while not STOP_EVENT.is_set():
         try:
+            sync_started_config_marker = file_marker(OPENCLAW_CONFIG_FILE)
             last_fingerprint, last_marker = sync_once(last_fingerprint, last_marker)
+            config_marker = file_marker(OPENCLAW_CONFIG_FILE)
+
+            if config_marker != sync_started_config_marker:
+                trigger, config_marker = wait_for_config_settle(config_marker)
+                if trigger == "stopped":
+                    break
+                print("OpenClaw config changed during sync; syncing again after it settled.")
+                continue
         except Exception as exc:
             write_status("error", f"Sync failed: {exc}")
             print(f"Workspace sync failed: {exc}")
+            config_marker = file_marker(OPENCLAW_CONFIG_FILE)
 
-        if STOP_EVENT.wait(INTERVAL):
+        trigger, config_marker = wait_for_sync_trigger(config_marker)
+        if trigger == "stopped":
             break
+        if trigger == "settled":
+            print("OpenClaw config changed and settled; syncing immediately.")
 
     return 0
 
@@ -469,6 +598,17 @@ def main() -> int:
             write_status("error", f"Shutdown sync failed: {exc}")
             print(f"Workspace sync: shutdown sync failed: {exc}")
             return 1
+    if command == "sync-once-settled":
+        try:
+            trigger, _ = wait_for_config_settle(file_marker(OPENCLAW_CONFIG_FILE))
+            if trigger == "stopped":
+                return 1
+            sync_once()
+            return 0
+        except Exception as exc:
+            write_status("error", f"Settled sync failed: {exc}")
+            print(f"Workspace sync: settled sync failed: {exc}")
+            return 1
     if command == "loop":
         return loop()
 

start.sh

@@ -11,6 +11,14 @@ umask 0077
 OPENCLAW_VERSION="${OPENCLAW_VERSION:-latest}"
 OPENCLAW_APP_DIR="/home/node/.openclaw/openclaw-app"
 OPENCLAW_RUNTIME_VERSION=""
+OPENCLAW_FILE_LOG_LEVEL_CONFIGURED=false
+OPENCLAW_CONSOLE_LOG_LEVEL_CONFIGURED=false
+OPENCLAW_CONSOLE_LOG_STYLE_CONFIGURED=false
+WHATSAPP_ENABLED_CONFIGURED=false
+[ "${OPENCLAW_FILE_LOG_LEVEL+x}" = "x" ] && OPENCLAW_FILE_LOG_LEVEL_CONFIGURED=true
+[ "${OPENCLAW_CONSOLE_LOG_LEVEL+x}" = "x" ] && OPENCLAW_CONSOLE_LOG_LEVEL_CONFIGURED=true
+[ "${OPENCLAW_CONSOLE_LOG_STYLE+x}" = "x" ] && OPENCLAW_CONSOLE_LOG_STYLE_CONFIGURED=true
+[ "${WHATSAPP_ENABLED+x}" = "x" ] && WHATSAPP_ENABLED_CONFIGURED=true
 WHATSAPP_ENABLED="${WHATSAPP_ENABLED:-false}"
 WHATSAPP_ENABLED_NORMALIZED=$(printf '%s' "$WHATSAPP_ENABLED" | tr '[:upper:]' '[:lower:]')
 SYNC_INTERVAL="${SYNC_INTERVAL:-180}"
@@ -445,22 +453,32 @@ if [ -f "$EXISTING_CONFIG" ]; then
     --arg consoleLevel "$OPENCLAW_CONSOLE_LOG_LEVEL" \
     --arg consoleStyle "$OPENCLAW_CONSOLE_LOG_STYLE" \
     --argjson desired "$CONFIG_JSON" \
+    --argjson fileLogConfigured "$OPENCLAW_FILE_LOG_LEVEL_CONFIGURED" \
+    --argjson consoleLogConfigured "$OPENCLAW_CONSOLE_LOG_LEVEL_CONFIGURED" \
+    --argjson consoleStyleConfigured "$OPENCLAW_CONSOLE_LOG_STYLE_CONFIGURED" \
+    --argjson whatsappConfigured "$WHATSAPP_ENABLED_CONFIGURED" \
     --argjson whatsappEnabled "$WHATSAPP_CONFIG_ENABLED" \
-    '.gateway.auth.token = $token
+    '(.channels.whatsapp // {}) as $existingWhatsapp
+     | .gateway.auth.token = $token
      | .agents.defaults.model = $model
-     | .logging.level = $fileLevel
-     | .logging.consoleLevel = $consoleLevel
-     | .logging.consoleStyle = $consoleStyle
+     | if $fileLogConfigured then .logging.level = $fileLevel else . end
+     | if $consoleLogConfigured then .logging.consoleLevel = $consoleLevel else . end
+     | if $consoleStyleConfigured then .logging.consoleStyle = $consoleStyle else . end
      | .channels = ((.channels // {}) * ($desired.channels // {}))
      | .plugins.allow = (((.plugins.allow // []) + ($desired.plugins.allow // [])) | unique)
      | .plugins.deny = (((.plugins.deny // []) + ($desired.plugins.deny // [])) | unique)
-     | .plugins.entries = ((.plugins.entries // {}) * ($desired.plugins.entries // {}))
+     | .plugins.entries = (($desired.plugins.entries // {}) * (.plugins.entries // {}))
      | if $whatsappEnabled then
-         .plugins.entries.whatsapp.enabled = true
-         | .channels.whatsapp = ($desired.channels.whatsapp // {"dmPolicy": "pairing"})
-       else
+         ($desired.channels.whatsapp // {"dmPolicy": "pairing"}) as $desiredWhatsapp
+         | .plugins.entries.whatsapp.enabled = true
+         | .channels.whatsapp = (($existingWhatsapp * $desiredWhatsapp)
+             | if ($existingWhatsapp | has("dmPolicy")) then .dmPolicy = $existingWhatsapp.dmPolicy else . end
+             | if ($existingWhatsapp | has("allowFrom")) then .allowFrom = $existingWhatsapp.allowFrom else . end)
+       elif $whatsappConfigured then
          .plugins.entries.whatsapp.enabled = false
          | del(.channels.whatsapp)
+       else
+         .
        end' \
     "$EXISTING_CONFIG" 2>/dev/null)
 
@@ -1000,59 +1018,108 @@ hc_finish_startup_commands
 sync_installed_plugins_into_allow
 
 # ── Launch gateway ──
-echo "Launching OpenClaw gateway on port 7860..."
+GATEWAY_RESTART_DELAY="${GATEWAY_RESTART_DELAY:-2}"
+GATEWAY_MAX_RESTARTS="${GATEWAY_MAX_RESTARTS:-0}"
+GATEWAY_RESTART_COUNT=0
+SYNC_LOOP_PID=""
+GUARDIAN_PID=""
+
+sync_before_gateway_restart() {
+  [ -n "${HF_TOKEN:-}" ] || return 0
+  [ -f "/home/node/app/openclaw-sync.py" ] || return 0
+
+  echo "Gateway stopped; saving latest OpenClaw state before restart..."
+  python3 /home/node/app/openclaw-sync.py sync-once-settled || \
+    echo "Warning: could not sync settled state before gateway restart"
+}
 
-GATEWAY_ARGS=(gateway run --port 7860 --bind lan)
-if [ "${GATEWAY_VERBOSE:-0}" = "1" ]; then
-  GATEWAY_ARGS+=(--verbose)
-  echo "Gateway verbose logging enabled (GATEWAY_VERBOSE=1)"
-fi
+start_background_sync_once() {
+  [ -n "${HF_TOKEN:-}" ] || return 0
 
-# Use stdbuf -oL -eL to ensure logs are not buffered and appear immediately
-# in the console. NOTE: $! captures the LAST pipeline element (tee), not
-# openclaw — fine for passing to `wait` (waits for the whole pipeline to
-# finish), but kill -0 on it is uninformative. We probe TCP instead.
-stdbuf -oL -eL openclaw "${GATEWAY_ARGS[@]}" 2>&1 | tee -a /home/node/.openclaw/gateway.log &
-GATEWAY_PID=$!
-
-# Poll for the gateway to start listening on 7860. OpenClaw can take 20-30s
-# on cold start (plugin install + auto-restore). Bail out early if the
-# pipeline died.
-GATEWAY_READY_TIMEOUT="${GATEWAY_READY_TIMEOUT:-90}"
-ready=false
-for ((i=0; i<GATEWAY_READY_TIMEOUT; i++)); do
-  if (echo > /dev/tcp/127.0.0.1/7860) 2>/dev/null; then
-    ready=true
-    break
-  fi
-  if ! kill -0 "$GATEWAY_PID" 2>/dev/null; then
-    break
+  if [ -n "$SYNC_LOOP_PID" ] && kill -0 "$SYNC_LOOP_PID" 2>/dev/null; then
+    return 0
   fi
-  sleep 1
-done
 
-if [ "$ready" != "true" ]; then
-  echo ""
-  echo "Gateway failed to start. Last 30 lines of log:"
-  echo "────────────────────────────────────────────"
-  tail -30 /home/node/.openclaw/gateway.log
-  exit 1
-fi
+  python3 -u /home/node/app/openclaw-sync.py loop &
+  SYNC_LOOP_PID=$!
+}
+
+start_guardian_once() {
+  [ "$WHATSAPP_ENABLED_NORMALIZED" = "true" ] || return 0
+
+  if [ -n "$GUARDIAN_PID" ] && kill -0 "$GUARDIAN_PID" 2>/dev/null; then
+    return 0
+  fi
 
-# 11. Start WhatsApp Guardian after the gateway is accepting connections
-if [ "$WHATSAPP_ENABLED_NORMALIZED" = "true" ]; then
   node /home/node/app/wa-guardian.js &
   GUARDIAN_PID=$!
   echo "WhatsApp Guardian started (PID: $GUARDIAN_PID)"
-fi
+}
 
-# 11.5 Warm up the managed browser so first browser actions have a live tab
-warmup_browser
+while true; do
+  echo "Launching OpenClaw gateway on port 7860..."
 
-# 12. Start Workspace Sync after startup settles
-if [ -n "${HF_TOKEN:-}" ]; then
-  python3 -u /home/node/app/openclaw-sync.py loop &
-fi
+  GATEWAY_ARGS=(gateway run --port 7860 --bind lan)
+  if [ "${GATEWAY_VERBOSE:-0}" = "1" ]; then
+    GATEWAY_ARGS+=(--verbose)
+    echo "Gateway verbose logging enabled (GATEWAY_VERBOSE=1)"
+  fi
+
+  # Use stdbuf -oL -eL to ensure logs are not buffered and appear immediately
+  # in the console. NOTE: $! captures the LAST pipeline element (tee), not
+  # openclaw — fine for passing to `wait` (waits for the whole pipeline to
+  # finish), but kill -0 on it is uninformative. We probe TCP instead.
+  stdbuf -oL -eL openclaw "${GATEWAY_ARGS[@]}" 2>&1 | tee -a /home/node/.openclaw/gateway.log &
+  GATEWAY_PID=$!
+
+  # Poll for the gateway to start listening on 7860. OpenClaw can take 20-30s
+  # on cold start (plugin install + auto-restore). Bail out early if the
+  # pipeline died.
+  GATEWAY_READY_TIMEOUT="${GATEWAY_READY_TIMEOUT:-90}"
+  ready=false
+  for ((i=0; i<GATEWAY_READY_TIMEOUT; i++)); do
+    if (echo > /dev/tcp/127.0.0.1/7860) 2>/dev/null; then
+      ready=true
+      break
+    fi
+    if ! kill -0 "$GATEWAY_PID" 2>/dev/null; then
+      break
+    fi
+    sleep 1
+  done
+
+  if [ "$ready" != "true" ]; then
+    echo ""
+    echo "Gateway failed to start. Last 30 lines of log:"
+    echo "────────────────────────────────────────────"
+    tail -30 /home/node/.openclaw/gateway.log
+    exit 1
+  fi
 
-# Wait for gateway (allows trap to fire)
-wait $GATEWAY_PID
+  # 11. Start WhatsApp Guardian after the gateway is accepting connections
+  start_guardian_once
+
+  # 11.5 Warm up the managed browser so first browser actions have a live tab
+  warmup_browser
+
+  # 12. Start Workspace Sync after startup settles. Keep only one loop active;
+  # config edits can make OpenClaw exit/reload, and the gateway loop below will
+  # relaunch it without rerunning all startup code.
+  start_background_sync_once
+
+  set +e
+  wait "$GATEWAY_PID"
+  GATEWAY_EXIT_CODE=$?
+  set -e
+
+  sync_before_gateway_restart
+
+  GATEWAY_RESTART_COUNT=$((GATEWAY_RESTART_COUNT + 1))
+  if [ "$GATEWAY_MAX_RESTARTS" != "0" ] && [ "$GATEWAY_RESTART_COUNT" -ge "$GATEWAY_MAX_RESTARTS" ]; then
+    echo "Gateway exited with code ${GATEWAY_EXIT_CODE}; restart limit (${GATEWAY_MAX_RESTARTS}) reached."
+    exit "$GATEWAY_EXIT_CODE"
+  fi
+
+  echo "Gateway exited with code ${GATEWAY_EXIT_CODE}; restarting in ${GATEWAY_RESTART_DELAY}s..."
+  sleep "$GATEWAY_RESTART_DELAY"
+done