Prune stale dataset files after sync upload

.env.example

@@ -132,7 +132,10 @@ LLM_MODEL=anthropic/claude-sonnet-4-5
 # have multiple accounts or want to spread rate-limit quota.
 #
 # Pattern: <PROVIDER>_API_KEYS=key1,key2,key3
-# Fallback order: plural pool → singular key → LLM_API_KEY
+# Fallback order: plural pool → singular key → LLM_API_KEY (optional)
+# Set false only if you want to disable global LLM_API_KEY fallback
+# across providers.
+LLM_API_KEY_FALLBACK_ENABLED=true
 #
 # Uncomment and fill in only the providers you use:
 #
@@ -172,6 +175,13 @@ LLM_MODEL=anthropic/claude-sonnet-4-5
 #      is also picked up by OpenClaw → that provider's models become
 #      available in the Control UI for manual selection.
 #   3. Rotation pools (*_API_KEYS) work for every active provider
+#
+# Optional: explicitly pin model lists per provider for Control UI visibility
+# when provider keys are configured.
+# Format: comma-separated model IDs
+# NVIDIA_MODELS=meta/llama-3.1-70b-instruct,nvidia/llama-3.1-nemotron-70b-instruct
+# OPENAI_MODELS=gpt-4o-mini,gpt-4.1
+# GROQ_MODELS=llama-3.3-70b-versatile,deepseek-r1-distill-llama-70b
 #      independently and in parallel.
 #
 # EXAMPLE — default Anthropic, also use OpenAI and Groq:

CHANGELOG.md

@@ -2,6 +2,44 @@
 
 All notable changes to this project will be documented in this file.
 
+## [1.5.0] - 2026-05-13
+
+### Added
+
+- **NVIDIA key-rotation support** — added `nvidia-key-rotator.cjs` wiring and startup integration so deployments can rotate NVIDIA credentials similarly to other provider key-rotation flows.
+- **Cloudflare keep-alive automation** — added/expanded `cloudflare-keepalive-setup.py` flow and startup wiring to provision keep-alive through Cloudflare Worker automation instead of the older UptimeRobot-first approach.
+- **Sync metadata marker model** — introduced a structured workspace marker `(file_count, total_size, newest_mtime, metadata_hash)` to support stronger change introspection in sync code.
+
+### Changed
+
+- **Workspace sync script rename finalized** — `workspace-sync.py` flow was migrated to `openclaw-sync.py` in Docker/startup/docs so restore/sync behavior is centralized under one script.
+- **Sync trigger behavior hardened for config churn** — OpenClaw config sync now debounces until JSON settles before immediate sync, reducing false/partial syncs during rapid config writes.
+- **Gateway restart flow now saves state first** — restart path was updated to run a pre-restart one-shot state sync so gateway reloads are less likely to drop recent state.
+- **Shutdown backup now uses a two-step pass** — graceful shutdown now attempts `sync-once-settled` then a final `sync-once` pass to better capture last-second writes.
+- **Telegram allowlist simplified** — consolidated Telegram allowlist into `TELEGRAM_ALLOWED_USERS` and aligned docs/examples.
+- **Plugin startup behavior aligned** — startup-installed plugins are synced into `plugins.allow` before gateway launch so runtime-installed plugins are recognized cleanly.
+- **Cloudflare proxy path matured** — multiple iterations improved fetch/proxy behavior (header handling, endpoint scoping, API root routing, URL parsing, and logging noise reduction), then simplified unstable undici patching paths.
+- **Health dashboard polish** — sync timestamps now show local time, footer credits were corrected, and status rendering/docs were updated for the Cloudflare keep-alive model.
+- **CI workflow churn documented** — GitHub workflow files for HF sync were added/renamed/cleaned multiple times as space/repo naming stabilized.
+
+### Fixed
+
+- **Missed rapid backup updates** — sync logic now relies on content fingerprint checks for no-op decisions so same-second or quick successive changes are less likely to be skipped.
+- **Non-deterministic metadata hashing** — metadata hashing now iterates paths deterministically to avoid hash jitter from traversal order.
+- **Transient file race sync failures** — sync fingerprinting/snapshot copy paths now tolerate transient `OSError` (file rotated/deleted mid-scan) instead of aborting the whole sync pass.
+- **State restore migration edge cases** — restore flow includes migration/cleanup behavior for legacy hidden state paths and stale backup entries.
+- **Startup/env robustness** — fixed shell export formatting/syntax issues (e.g., NVIDIA/XAI lines) and unbound-variable pitfalls in startup scripts.
+- **Proxy runtime errors and noise** — fixed specific proxy runtime issues (including `UND_ERR_INVALID_ARG`, fetch duplex handling, and upstream error visibility) and reduced noisy stdout logs that interfered with clean process output.
+- **HF workflow/repo reference mismatches** — corrected and later cleaned workflow repository references during repo migration/restructure.
+
+### Docs
+
+- README/.env/security docs were refreshed across multiple commits to reflect:
+  - Cloudflare keep-alive replacing UptimeRobot setup path,
+  - updated secrets and startup environment behavior,
+  - provider/key-rotation options,
+  - backup/sync behavior and troubleshooting guidance.
+
 ## [1.4.0] - 2026-04-25
 
 ### Added

ENV_VARIABLES_FULL_LIST.md

@@ -0,0 +1,254 @@
+# ENV Variables Full List (Repo Scope)
+
+> Internal reference file. Not linked from README/docs navigation.
+
+## 1) User-settable (Space Secrets / `.env`)
+
+These are the main variables you can set directly.
+
+### Core startup/auth
+- `LLM_API_KEY` — string API key. Example: `LLM_API_KEY=sk-ant-xxxx`
+- `LLM_MODEL` — `provider/model-id` string. Examples:
+  - `LLM_MODEL=anthropic/claude-sonnet-4-5`
+  - `LLM_MODEL=nvidia/meta/llama-3.1-70b-instruct`
+- `GATEWAY_TOKEN` — random secret string. Example: `GATEWAY_TOKEN=9d2f...`
+- `OPENCLAW_PASSWORD` — plaintext password string. Example: `OPENCLAW_PASSWORD=MyStrongPass123`
+
+### Provider fallback/selection
+- `LLM_API_KEY_FALLBACK_ENABLED` — boolean-like:
+  - enabled: `true`, `1`, `yes`, `on`
+  - disabled: `false`, `0`, `no`, `off`
+  - default/example: `LLM_API_KEY_FALLBACK_ENABLED=true`
+
+### Provider-specific keys (single-key form)
+Use plain provider key strings. Examples:
+- `NVIDIA_API_KEY=nvapi-xxxx`
+- `OPENAI_API_KEY=sk-xxxx`
+- `GEMINI_API_KEY=AIzaSyxxxx`
+
+Full list:
+- `ANTHROPIC_API_KEY`
+- `OPENAI_API_KEY`
+- `GEMINI_API_KEY`
+- `DEEPSEEK_API_KEY`
+- `OPENROUTER_API_KEY`
+- `KILOCODE_API_KEY`
+- `OPENCODE_API_KEY`
+- `ZAI_API_KEY`
+- `MOONSHOT_API_KEY`
+- `KIMI_API_KEY`
+- `MINIMAX_API_KEY`
+- `MODELSTUDIO_API_KEY`
+- `XIAOMI_API_KEY`
+- `VOLCANO_ENGINE_API_KEY`
+- `BYTEPLUS_API_KEY`
+- `QIANFAN_API_KEY`
+- `MISTRAL_API_KEY`
+- `XAI_API_KEY`
+- `NVIDIA_API_KEY`
+- `COHERE_API_KEY`
+- `GROQ_API_KEY`
+- `TOGETHER_API_KEY`
+- `CEREBRAS_API_KEY`
+- `VENICE_API_KEY`
+- `SYNTHETIC_API_KEY`
+- `COPILOT_GITHUB_TOKEN`
+- `HUGGINGFACE_HUB_TOKEN`
+- `AI_GATEWAY_API_KEY`
+
+### Provider key pools (comma-separated rotation)
+Format: comma-separated keys, no spaces required. Examples:
+- `NVIDIA_API_KEYS=nvapi-key1,nvapi-key2`
+- `OPENAI_API_KEYS=sk-a,sk-b,sk-c`
+
+Full list:
+- `ANTHROPIC_API_KEYS`
+- `OPENAI_API_KEYS`
+- `GEMINI_API_KEYS`
+- `DEEPSEEK_API_KEYS`
+- `OPENROUTER_API_KEYS`
+- `KILOCODE_API_KEYS`
+- `OPENCODE_API_KEYS`
+- `ZAI_API_KEYS`
+- `MOONSHOT_API_KEYS`
+- `MINIMAX_API_KEYS`
+- `XIAOMI_API_KEYS`
+- `VOLCANO_ENGINE_API_KEYS`
+- `BYTEPLUS_API_KEYS`
+- `QIANFAN_API_KEYS`
+- `MODELSTUDIO_API_KEYS`
+- `KIMI_API_KEYS`
+- `MISTRAL_API_KEYS`
+- `XAI_API_KEYS`
+- `NVIDIA_API_KEYS`
+- `GROQ_API_KEYS`
+- `COHERE_API_KEYS`
+- `TOGETHER_API_KEYS`
+- `CEREBRAS_API_KEYS`
+- `HUGGINGFACE_HUB_TOKENS`
+
+### Provider model lists (optional, for Control UI visibility)
+Format: comma-separated model IDs; used only when that provider has key(s) configured.
+Examples:
+- `NVIDIA_MODELS=meta/llama-3.1-70b-instruct,nvidia/llama-3.1-nemotron-70b-instruct`
+- `OPENAI_MODELS=gpt-4o-mini,gpt-4.1`
+- `GROQ_MODELS=llama-3.3-70b-versatile,deepseek-r1-distill-llama-70b`
+
+Supported:
+- `ANTHROPIC_MODELS`
+- `OPENAI_MODELS`
+- `GEMINI_MODELS`
+- `DEEPSEEK_MODELS`
+- `OPENROUTER_MODELS`
+- `KILOCODE_MODELS`
+- `OPENCODE_MODELS`
+- `ZAI_MODELS`
+- `MOONSHOT_MODELS`
+- `KIMI_MODELS`
+- `MINIMAX_MODELS`
+- `MODELSTUDIO_MODELS`
+- `XIAOMI_MODELS`
+- `VOLCANO_ENGINE_MODELS`
+- `BYTEPLUS_MODELS`
+- `QIANFAN_MODELS`
+- `GROQ_MODELS`
+- `MISTRAL_MODELS`
+- `XAI_MODELS`
+- `NVIDIA_MODELS`
+- `COHERE_MODELS`
+- `TOGETHER_MODELS`
+- `CEREBRAS_MODELS`
+
+### Telegram / WhatsApp
+- `TELEGRAM_BOT_TOKEN` — bot token string. Example: `TELEGRAM_BOT_TOKEN=123456:ABC...`
+- `TELEGRAM_ALLOWED_USERS` — comma-separated numeric IDs. Example: `TELEGRAM_ALLOWED_USERS=12345,67890`
+- `TELEGRAM_USER_ID` — legacy single numeric ID. Example: `TELEGRAM_USER_ID=12345`
+- `TELEGRAM_USER_IDS` — legacy comma-separated IDs. Example: `TELEGRAM_USER_IDS=12345,67890`
+- `WHATSAPP_ENABLED` — boolean-like (`true/false`, `1/0`, `yes/no`, `on/off`). Example: `WHATSAPP_ENABLED=true`
+
+### Backup/sync
+- `HF_TOKEN` — HF token string. Example: `HF_TOKEN=hf_xxxxx`
+- `HF_USERNAME` — HF username string. Example: `HF_USERNAME=myhandle`
+- `BACKUP_DATASET_NAME` — dataset slug string. Example: `BACKUP_DATASET_NAME=huggingclaw-backup`
+- `SYNC_INTERVAL` — integer seconds. Example: `SYNC_INTERVAL=180`
+- `SYNC_START_DELAY` — integer seconds. Example: `SYNC_START_DELAY=10`
+- `SYNC_MAX_FILE_BYTES` — integer bytes. Example: `SYNC_MAX_FILE_BYTES=52428800`
+- `OPENCLAW_CONFIG_WATCH_INTERVAL` — number (supports decimal seconds). Example: `OPENCLAW_CONFIG_WATCH_INTERVAL=1`
+- `OPENCLAW_CONFIG_SETTLE_SECONDS` — number (supports decimal seconds). Example: `OPENCLAW_CONFIG_SETTLE_SECONDS=3`
+
+### Gateway/runtime tuning
+- `GATEWAY_HOST` — hostname/IP string. Example: `GATEWAY_HOST=127.0.0.1`
+- `GATEWAY_PORT` — integer port. Example: `GATEWAY_PORT=3456`
+- `GATEWAY_VERBOSE` — boolean-like. Example: `GATEWAY_VERBOSE=true`
+- `GATEWAY_READY_TIMEOUT` — integer seconds. Example: `GATEWAY_READY_TIMEOUT=60`
+- `GATEWAY_MAX_RESTARTS` — integer count. Example: `GATEWAY_MAX_RESTARTS=10`
+- `GATEWAY_RESTART_DELAY` — integer seconds. Example: `GATEWAY_RESTART_DELAY=2`
+- `TRUSTED_PROXIES` — comma-separated CIDR/IP list. Example: `TRUSTED_PROXIES=127.0.0.1,10.0.0.0/8`
+- `ALLOWED_ORIGINS` — comma-separated origins. Example: `ALLOWED_ORIGINS=https://a.com,https://b.com`
+- `PORT` — integer external port. Example: `PORT=7860`
+
+### Browser/plugin controls
+- `BROWSER_PLUGIN_MODE` — mode string (`auto`, `builtin`, `external`, etc. depending on runtime). Example: `BROWSER_PLUGIN_MODE=auto`
+- `BROWSER_EXECUTABLE_PATH` — absolute path. Example: `BROWSER_EXECUTABLE_PATH=/usr/bin/chromium`
+- `BROWSER_DISABLED` — boolean-like. Example: `BROWSER_DISABLED=false`
+- `ACP_PLUGIN_MODE` — mode string. Example: `ACP_PLUGIN_MODE=auto`
+- `ACPX_DISABLED` — boolean-like. Example: `ACPX_DISABLED=true`
+
+### OpenClaw logging/behavior
+- `OPENCLAW_VERSION` — image/app version string. Example: `OPENCLAW_VERSION=latest`
+- `OPENCLAW_RUNTIME_VERSION` — runtime tag string. Example: `OPENCLAW_RUNTIME_VERSION=v1.2.3`
+- `OPENCLAW_DISPLAY_VERSION` — display label string. Example: `OPENCLAW_DISPLAY_VERSION=Custom`
+- `OPENCLAW_DISABLE_BONJOUR` — boolean-like. Example: `OPENCLAW_DISABLE_BONJOUR=true`
+- `OPENCLAW_CONSOLE_LOG_LEVEL` — log level (`trace|debug|info|warn|error`). Example: `OPENCLAW_CONSOLE_LOG_LEVEL=warn`
+- `OPENCLAW_CONSOLE_LOG_STYLE` — style string. Example: `OPENCLAW_CONSOLE_LOG_STYLE=pretty`
+- `OPENCLAW_FILE_LOG_LEVEL` — log level (`trace|debug|info|warn|error`). Example: `OPENCLAW_FILE_LOG_LEVEL=info`
+
+### Custom OpenAI-compatible provider
+- `CUSTOM_PROVIDER_NAME` — unique provider prefix. Example: `CUSTOM_PROVIDER_NAME=modal`
+- `CUSTOM_BASE_URL` — base URL only (not `/chat/completions`). Example: `CUSTOM_BASE_URL=https://api.modal.com/v1`
+- `CUSTOM_MODEL_ID` — model id string. Example: `CUSTOM_MODEL_ID=zai-org/GLM-5.1-FP8`
+- `CUSTOM_MODEL_NAME` — display name. Example: `CUSTOM_MODEL_NAME=GLM 5.1 FP8`
+- `CUSTOM_API_KEY` — provider key string. Example: `CUSTOM_API_KEY=sk-xxxx`
+- `CUSTOM_API_TYPE` — API mode string. Example: `CUSTOM_API_TYPE=openai-completions`
+- `CUSTOM_CONTEXT_WINDOW` — integer token window. Example: `CUSTOM_CONTEXT_WINDOW=128000`
+- `CUSTOM_MAX_TOKENS` — integer max tokens. Example: `CUSTOM_MAX_TOKENS=500`
+
+### Cloudflare proxy/keepalive
+- `CLOUDFLARE_WORKERS_TOKEN` — API token string. Example: `CLOUDFLARE_WORKERS_TOKEN=cf_xxx`
+- `CLOUDFLARE_WORKER_NAME` — worker name string. Example: `CLOUDFLARE_WORKER_NAME=huggingclaw-proxy`
+- `CLOUDFLARE_ACCOUNT_ID` — account id string. Example: `CLOUDFLARE_ACCOUNT_ID=abc123...`
+- `CLOUDFLARE_PROXY_URL` — full proxy URL. Example: `CLOUDFLARE_PROXY_URL=https://my-worker.workers.dev`
+- `CLOUDFLARE_PROXY_SECRET` — shared secret string. Example: `CLOUDFLARE_PROXY_SECRET=long-random-secret`
+- `CLOUDFLARE_PROXY_DOMAINS` — comma-separated domains. Example: `CLOUDFLARE_PROXY_DOMAINS=api.openai.com,api.anthropic.com`
+- `CLOUDFLARE_PROXY_DEBUG` — boolean-like. Example: `CLOUDFLARE_PROXY_DEBUG=false`
+- `CLOUDFLARE_KEEPALIVE_ENABLED` — boolean-like. Example: `CLOUDFLARE_KEEPALIVE_ENABLED=true`
+- `CLOUDFLARE_KEEPALIVE_URL` — URL string. Example: `CLOUDFLARE_KEEPALIVE_URL=https://space-url.hf.space`
+- `CLOUDFLARE_KEEPALIVE_CRON` — cron string. Example: `CLOUDFLARE_KEEPALIVE_CRON=*/5 * * * *`
+- `CLOUDFLARE_KEEPALIVE_WORKER_NAME` — worker name string. Example: `CLOUDFLARE_KEEPALIVE_WORKER_NAME=huggingclaw-keepalive`
+
+### Startup command replay/install helpers
+- `HUGGINGCLAW_RUN` — shell commands string. Example: `HUGGINGCLAW_RUN=apt-get update && apt-get install -y ffmpeg`
+- `HUGGINGCLAW_APT_PACKAGES` — comma/space separated package names. Example: `HUGGINGCLAW_APT_PACKAGES=ffmpeg,git`
+- `HUGGINGCLAW_PIP_PACKAGES` — pip package list string. Example: `HUGGINGCLAW_PIP_PACKAGES=yt-dlp,requests`
+- `HUGGINGCLAW_NPM_PACKAGES` — npm package list string. Example: `HUGGINGCLAW_NPM_PACKAGES=sharp,axios`
+- `HUGGINGCLAW_OPENCLAW_PLUGINS` — plugin ids list string. Example: `HUGGINGCLAW_OPENCLAW_PLUGINS=@openclaw/telegram,@openclaw/whatsapp`
+- `HUGGINGCLAW_STARTUP_SCRIPT` — shell script string. Example: `HUGGINGCLAW_STARTUP_SCRIPT=echo hello`
+- `HUGGINGCLAW_STARTUP_SCRIPT_B64` — base64 script content. Example: `HUGGINGCLAW_STARTUP_SCRIPT_B64=IyEvYmluL2Jhc2gKZWNobyBoaQ==`
+- `HUGGINGCLAW_STARTUP_COMMANDS` — newline/semicolon commands string.
+- `HUGGINGCLAW_STARTUP_STRICT` — boolean-like. Example: `HUGGINGCLAW_STARTUP_STRICT=true`
+- `WEBHOOK_URL` — webhook endpoint URL. Example: `WEBHOOK_URL=https://hooks.example.com/hc`
+
+---
+
+## 2) Runtime/Internal envs (normally do **not** set manually)
+
+These are mostly derived/temporary/process variables used by scripts:
+
+- `APP_BASE`
+- `BACKUP_DATASET`
+- `BROWSER_SHOULD_ENABLE`
+- `CF_PROXY_ENV_FILE`
+- `CLEAN_TG_TOKEN`
+- `CONFIG_JSON`
+- `CUSTOM_BASE_URL_NORMALIZED`
+- `CUSTOM_PROVIDER_NORMALIZED`
+- `CUSTOM_PROVIDER_OK`
+- `DEBIAN_FRONTEND`
+- `ERRORS`
+- `EXISTING_CONFIG`
+- `GATEWAY_EXIT_CODE`
+- `GATEWAY_PID`
+- `GATEWAY_RESTART_COUNT`
+- `GUARDIAN_PID`
+- `HC_STARTUP_FAILURES`
+- `HC_STARTUP_INDEX`
+- `HC_STARTUP_STRICT_NORMALIZED`
+- `HC_STARTUP_VAR`
+- `HOME`
+- `IDS_JSON`
+- `INSTALLS`
+- `LLM_PROVIDER`
+- `NODE_OPTIONS`
+- `NPM_CONFIG_PREFIX`
+- `OPENCLAW_APP_DIR`
+- `OPENCLAW_CONSOLE_LOG_LEVEL_CONFIGURED`
+- `OPENCLAW_CONSOLE_LOG_STYLE_CONFIGURED`
+- `OPENCLAW_FILE_LOG_LEVEL_CONFIGURED`
+- `ORIGINS_JSON`
+- `PATCHED`
+- `PATH`
+- `PLUGIN_ALLOW_JSON`
+- `PROXIES_JSON`
+- `PROXY_URL`
+- `PYTHONUSERBASE`
+- `RESET_MARKER_PATH`
+- `SPACE_AUTHOR_NAME`
+- `SPACE_HOST`
+- `SPACE_REPO_NAME`
+- `STARTUP_FILE`
+- `SYNC_LOOP_PID`
+- `VIRTUAL_ENV`
+- `WEBHOOK_BODY`
+- `WHATSAPP_CONFIG_ENABLED`
+- `WHATSAPP_ENABLED_CONFIGURED`
+- `WHATSAPP_ENABLED_NORMALIZED`

README.md

@@ -250,10 +250,10 @@ GEMINI_API_KEYS=AIza-key1,AIza-key2
 **Fallback chain** (per provider):
 1. `{PROVIDER}_API_KEYS` — comma-separated pool *(preferred)*
 2. `{PROVIDER}_API_KEY` — single dedicated key
-3. `LLM_API_KEY` — universal fallback *(default for all providers)*
+3. `LLM_API_KEY` — universal fallback *(enabled by default; disable with `LLM_API_KEY_FALLBACK_ENABLED=false`)*
 
 > [!TIP]
-> If you only set `LLM_API_KEY`, all providers use it as a fallback automatically — no extra config needed. Add per-provider pools only when you need multi-key rotation.
+> By default, `LLM_API_KEY` fallback is enabled for compatibility. Set `LLM_API_KEY_FALLBACK_ENABLED=false` if you want strict provider-only activation.
 
 Supported per-provider variables: `ANTHROPIC_API_KEYS`, `OPENAI_API_KEYS`, `GEMINI_API_KEYS`, `DEEPSEEK_API_KEYS`, `GROQ_API_KEYS`, `MISTRAL_API_KEYS`, `OPENROUTER_API_KEYS`, `XAI_API_KEYS`, `NVIDIA_API_KEYS`, `COHERE_API_KEYS`, `TOGETHER_API_KEYS`, `CEREBRAS_API_KEYS`, and more — see `.env.example` for the full list.
 

multi-provider-key-rotator.cjs

@@ -8,7 +8,7 @@
  *
  * For each provider you can supply a comma-separated pool:
  *   ANTHROPIC_API_KEYS=key1,key2,key3
- * Falls back to the singular env var, then to LLM_API_KEY.
+ * Falls back to the singular env var, and optionally to LLM_API_KEY.
  *
  * Keys are rotated round-robin per provider independently.
  *
@@ -30,7 +30,9 @@ const log = (...args) => console.error(...args);
 // envPlural – env var that holds a comma-separated key pool  (preferred)
 // envSingular – env var that holds a single key              (fallback)
 //
-// LLM_API_KEY is the final fallback for every provider.
+// LLM_API_KEY fallback can be controlled via:
+//   LLM_API_KEY_FALLBACK_ENABLED=true|false
+// Default is enabled for backwards compatibility.
 //
 const PROVIDERS = [
   {
@@ -180,6 +182,8 @@ function normalizeKeys(...inputs) {
 
 // Build per-provider key pools + rotation indices
 const providerState = PROVIDERS.map(p => {
+  const llmFallbackRaw = String(process.env.LLM_API_KEY_FALLBACK_ENABLED || '').trim().toLowerCase();
+  const llmFallbackEnabled = !/^(0|false|no|off)$/.test(llmFallbackRaw);
   const dedicatedKeys = normalizeKeys(
     process.env[p.envPlural]  || '',
     process.env[p.envSingular] || '',
@@ -187,7 +191,11 @@ const providerState = PROVIDERS.map(p => {
   const hasDedicated = dedicatedKeys.length > 0;
   const keys = hasDedicated
     ? dedicatedKeys
-    : normalizeKeys(process.env.LLM_API_KEY || '');
+    : (
+      llmFallbackEnabled
+        ? normalizeKeys(process.env.LLM_API_KEY || '')
+        : []
+    );
 
   if (hasDedicated) {
     log(`[key-rotator] ${p.name}: ${keys.length} key${keys.length === 1 ? '' : 's'}`);
@@ -208,6 +216,8 @@ const fallbackCount = providerState.filter(p => {
 }).length;
 if (fallbackCount > 0) {
   log(`[key-rotator] ${fallbackCount} provider(s) using LLM_API_KEY fallback`);
+} else if (process.env.LLM_API_KEY && /^(0|false|no|off)$/i.test(String(process.env.LLM_API_KEY_FALLBACK_ENABLED || ''))) {
+  log('[key-rotator] LLM_API_KEY fallback disabled (set LLM_API_KEY_FALLBACK_ENABLED=true to re-enable)');
 }
 
 // ─── Runtime helpers ─────────────────────────────────────────────────────────

openclaw-sync.py

@@ -18,6 +18,7 @@ import sys
 import tempfile
 import threading
 import time
+from typing import TypeAlias
 from pathlib import Path
 
 os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
@@ -69,6 +70,7 @@ EXCLUDED_STATE_NAMES = {
     "openclaw-app",
     "gateway.log",
     "browser",
+    "npm",
 }
 WHATSAPP_CREDS_DIR = OPENCLAW_HOME / "credentials" / "whatsapp" / "default"
 WHATSAPP_BACKUP_DIR = STATE_DIR / "credentials" / "whatsapp" / "default"
@@ -76,6 +78,7 @@ RESET_MARKER = WORKSPACE / ".reset_credentials"
 HF_API = HfApi(token=HF_TOKEN) if HF_TOKEN else None
 STOP_EVENT = threading.Event()
 _REPO_ID_CACHE: str | None = None
+WorkspaceMarker: TypeAlias = tuple[int, int, int, str]
 
 
 def write_status(status: str, message: str) -> None:
@@ -280,14 +283,15 @@ def file_marker(path: Path) -> tuple[int, int, int]:
     return (1, int(stat.st_size), int(stat.st_mtime_ns))
 
 
-def metadata_marker(root: Path) -> tuple[int, int, int]:
+def metadata_marker(root: Path) -> WorkspaceMarker:
     if not root.exists():
-        return (0, 0, 0)
+        return (0, 0, 0, "")
 
     file_count = 0
     total_size = 0
     newest_mtime = 0
-    for path in root.rglob("*"):
+    metadata_hasher = hashlib.sha256()
+    for path in sorted(root.rglob("*")):
         if not path.is_file():
             continue
         rel = path.relative_to(root).as_posix()
@@ -298,9 +302,17 @@ def metadata_marker(root: Path) -> tuple[int, int, int]:
         except OSError:
             continue
         file_count += 1
-        total_size += int(stat.st_size)
-        newest_mtime = max(newest_mtime, int(stat.st_mtime_ns))
-    return (file_count, total_size, newest_mtime)
+        size = int(stat.st_size)
+        mtime_ns = int(stat.st_mtime_ns)
+        total_size += size
+        newest_mtime = max(newest_mtime, mtime_ns)
+        metadata_hasher.update(rel.encode("utf-8"))
+        metadata_hasher.update(b"\0")
+        metadata_hasher.update(str(size).encode("ascii"))
+        metadata_hasher.update(b"\0")
+        metadata_hasher.update(str(mtime_ns).encode("ascii"))
+        metadata_hasher.update(b"\0")
+    return (file_count, total_size, newest_mtime, metadata_hasher.hexdigest())
 
 
 def fingerprint_dir(root: Path) -> str:
@@ -313,9 +325,16 @@ def fingerprint_dir(root: Path) -> str:
         if _should_exclude(rel, path):
             continue
         hasher.update(rel.encode("utf-8"))
-        with path.open("rb") as handle:
-            for chunk in iter(lambda: handle.read(1024 * 1024), b""):
-                hasher.update(chunk)
+        try:
+            with path.open("rb") as handle:
+                for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+                    hasher.update(chunk)
+        except (FileNotFoundError, IsADirectoryError, NotADirectoryError):
+            # Fingerprint must represent a complete view of the workspace.
+            # Retry next sync pass instead of silently hashing a partial tree.
+            raise RuntimeError(
+                f"Workspace changed while hashing {rel}; retrying next sync pass."
+            )
     return hasher.hexdigest()
 
 
@@ -331,10 +350,40 @@ def create_snapshot_dir(source_root: Path) -> Path:
             target.mkdir(parents=True, exist_ok=True)
             continue
         target.parent.mkdir(parents=True, exist_ok=True)
-        shutil.copy2(path, target)
+        try:
+            shutil.copy2(path, target)
+        except (FileNotFoundError, IsADirectoryError, NotADirectoryError):
+            # Do not upload a partial snapshot; let caller retry on next loop.
+            raise RuntimeError(
+                f"Snapshot changed while copying {rel_posix}; retrying next sync pass."
+            )
     return staging_root
 
 
+def prune_remote_deleted_files(repo_id: str, snapshot_dir: Path) -> None:
+    if HF_API is None:
+        return
+
+    local_files = {
+        path.relative_to(snapshot_dir).as_posix()
+        for path in snapshot_dir.rglob("*")
+        if path.is_file()
+    }
+
+    remote_files = HF_API.list_repo_files(repo_id=repo_id, repo_type="dataset")
+    stale_files = [
+        path for path in remote_files
+        if path not in local_files and path not in {".gitattributes"}
+    ]
+    for stale_path in stale_files:
+        HF_API.delete_file(
+            path_in_repo=stale_path,
+            repo_id=repo_id,
+            repo_type="dataset",
+            commit_message=f"Delete stale file {stale_path}",
+        )
+
+
 def restore_workspace() -> bool:
     if not HF_TOKEN:
         write_status("disabled", "HF_TOKEN is not configured.")
@@ -396,16 +445,15 @@ def restore_workspace() -> bool:
 
 def _sync_once_unlocked(
     last_fingerprint: str | None = None,
-    last_marker: tuple[int, int, int] | None = None,
-) -> tuple[str, tuple[int, int, int]]:
+    last_marker: WorkspaceMarker | None = None,
+) -> tuple[str, WorkspaceMarker]:
     if not HF_TOKEN:
         write_status("disabled", "HF_TOKEN is not configured.")
-        return (last_fingerprint or "", last_marker or (0, 0, 0))
+        return (last_fingerprint or "", last_marker or (0, 0, 0, ""))
 
     snapshot_state_into_workspace()
     repo_id = ensure_repo_exists()
     current_marker = metadata_marker(WORKSPACE)
-
     if last_marker is not None and current_marker == last_marker:
         write_status("synced", "No workspace changes detected.")
         return (last_fingerprint or "", current_marker)
@@ -435,6 +483,7 @@ def _sync_once_unlocked(
                 commit_message=f"HuggingClaw sync {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}",
                 ignore_patterns=[".git/*", ".git"],
             )
+        prune_remote_deleted_files(repo_id, snapshot_dir)
     finally:
         shutil.rmtree(snapshot_dir, ignore_errors=True)
 
@@ -444,8 +493,8 @@ def _sync_once_unlocked(
 
 def sync_once(
     last_fingerprint: str | None = None,
-    last_marker: tuple[int, int, int] | None = None,
-) -> tuple[str, tuple[int, int, int]]:
+    last_marker: WorkspaceMarker | None = None,
+) -> tuple[str, WorkspaceMarker]:
     SYNC_LOCK_FILE.parent.mkdir(parents=True, exist_ok=True)
     with SYNC_LOCK_FILE.open("w", encoding="utf-8") as lock_handle:
         fcntl.flock(lock_handle, fcntl.LOCK_EX)

start.sh

@@ -131,6 +131,48 @@ case "$LLM_PROVIDER" in
     ;;
 esac
 
+# Ensure OpenClaw provider discovery can see per-provider keys even when users
+# configure only *_API_KEYS pools. Mirror first pool key into singular env.
+promote_first_pool_key() {
+  local singular_var="$1"
+  local pool_var="$2"
+  local singular_val="${!singular_var:-}"
+  local pool_val="${!pool_var:-}"
+
+  [ -n "$singular_val" ] && return 0
+  [ -n "$pool_val" ] || return 0
+
+  local first
+  first=$(printf '%s' "$pool_val" | tr ',' '\n' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | awk 'NF{print; exit}')
+  [ -n "$first" ] || return 0
+  export "${singular_var}=$first"
+}
+
+promote_first_pool_key "ANTHROPIC_API_KEY" "ANTHROPIC_API_KEYS"
+promote_first_pool_key "OPENAI_API_KEY" "OPENAI_API_KEYS"
+promote_first_pool_key "GEMINI_API_KEY" "GEMINI_API_KEYS"
+promote_first_pool_key "DEEPSEEK_API_KEY" "DEEPSEEK_API_KEYS"
+promote_first_pool_key "OPENROUTER_API_KEY" "OPENROUTER_API_KEYS"
+promote_first_pool_key "KILOCODE_API_KEY" "KILOCODE_API_KEYS"
+promote_first_pool_key "OPENCODE_API_KEY" "OPENCODE_API_KEYS"
+promote_first_pool_key "ZAI_API_KEY" "ZAI_API_KEYS"
+promote_first_pool_key "MOONSHOT_API_KEY" "MOONSHOT_API_KEYS"
+promote_first_pool_key "MINIMAX_API_KEY" "MINIMAX_API_KEYS"
+promote_first_pool_key "XIAOMI_API_KEY" "XIAOMI_API_KEYS"
+promote_first_pool_key "VOLCANO_ENGINE_API_KEY" "VOLCANO_ENGINE_API_KEYS"
+promote_first_pool_key "BYTEPLUS_API_KEY" "BYTEPLUS_API_KEYS"
+promote_first_pool_key "QIANFAN_API_KEY" "QIANFAN_API_KEYS"
+promote_first_pool_key "MODELSTUDIO_API_KEY" "MODELSTUDIO_API_KEYS"
+promote_first_pool_key "KIMI_API_KEY" "KIMI_API_KEYS"
+promote_first_pool_key "MISTRAL_API_KEY" "MISTRAL_API_KEYS"
+promote_first_pool_key "XAI_API_KEY" "XAI_API_KEYS"
+promote_first_pool_key "NVIDIA_API_KEY" "NVIDIA_API_KEYS"
+promote_first_pool_key "GROQ_API_KEY" "GROQ_API_KEYS"
+promote_first_pool_key "COHERE_API_KEY" "COHERE_API_KEYS"
+promote_first_pool_key "TOGETHER_API_KEY" "TOGETHER_API_KEYS"
+promote_first_pool_key "CEREBRAS_API_KEY" "CEREBRAS_API_KEYS"
+promote_first_pool_key "HUGGINGFACE_HUB_TOKEN" "HUGGINGFACE_HUB_TOKENS"
+
 # ── Setup directories ──
 mkdir -p /home/node/.openclaw/agents/main/sessions
 mkdir -p /home/node/.openclaw/credentials
@@ -285,6 +327,67 @@ if [ -n "$CUSTOM_PROVIDER_NAME" ] || [ -n "$CUSTOM_BASE_URL" ] || [ -n "$CUSTOM_
   fi
 fi
 
+# Optional: explicitly expose provider model lists in Control UI when
+# provider keys are configured. Format:
+#   NVIDIA_MODELS=model1,model2
+#   OPENAI_MODELS=gpt-4o-mini,gpt-4.1
+# This helps when provider auto-discovery does not populate models reliably.
+inject_provider_models_from_env() {
+  local provider="$1"
+  local models_env="$2"
+  local key_env_single="$3"
+  local key_env_pool="$4"
+  local models_csv="${!models_env:-}"
+  local single_key="${!key_env_single:-}"
+  local pool_keys="${!key_env_pool:-}"
+
+  # Only inject when both:
+  # 1) provider has at least one configured key
+  # 2) explicit model list env is provided
+  if [ -z "$models_csv" ] || { [ -z "$single_key" ] && [ -z "$pool_keys" ]; }; then
+    return 0
+  fi
+
+  local models_json
+  models_json=$(printf '%s' "$models_csv" \
+    | tr ',' '\n' \
+    | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' \
+    | awk 'NF' \
+    | jq -R . \
+    | jq -s 'map({id: ., name: .}) | unique_by(.id)')
+
+  CONFIG_JSON=$(jq \
+    --arg provider "$provider" \
+    --argjson models "$models_json" \
+    '.models.mode = "merge"
+     | .models.providers[$provider] = ((.models.providers[$provider] // {}) + {models: $models})' <<<"$CONFIG_JSON")
+}
+
+# Built-in provider model envs (optional)
+inject_provider_models_from_env "anthropic" "ANTHROPIC_MODELS" "ANTHROPIC_API_KEY" "ANTHROPIC_API_KEYS"
+inject_provider_models_from_env "openai" "OPENAI_MODELS" "OPENAI_API_KEY" "OPENAI_API_KEYS"
+inject_provider_models_from_env "google" "GEMINI_MODELS" "GEMINI_API_KEY" "GEMINI_API_KEYS"
+inject_provider_models_from_env "deepseek" "DEEPSEEK_MODELS" "DEEPSEEK_API_KEY" "DEEPSEEK_API_KEYS"
+inject_provider_models_from_env "openrouter" "OPENROUTER_MODELS" "OPENROUTER_API_KEY" "OPENROUTER_API_KEYS"
+inject_provider_models_from_env "kilocode" "KILOCODE_MODELS" "KILOCODE_API_KEY" "KILOCODE_API_KEYS"
+inject_provider_models_from_env "opencode" "OPENCODE_MODELS" "OPENCODE_API_KEY" "OPENCODE_API_KEYS"
+inject_provider_models_from_env "zai" "ZAI_MODELS" "ZAI_API_KEY" "ZAI_API_KEYS"
+inject_provider_models_from_env "moonshot" "MOONSHOT_MODELS" "MOONSHOT_API_KEY" "MOONSHOT_API_KEYS"
+inject_provider_models_from_env "kimi-coding" "KIMI_MODELS" "KIMI_API_KEY" "KIMI_API_KEYS"
+inject_provider_models_from_env "minimax" "MINIMAX_MODELS" "MINIMAX_API_KEY" "MINIMAX_API_KEYS"
+inject_provider_models_from_env "modelstudio" "MODELSTUDIO_MODELS" "MODELSTUDIO_API_KEY" "MODELSTUDIO_API_KEYS"
+inject_provider_models_from_env "xiaomi" "XIAOMI_MODELS" "XIAOMI_API_KEY" "XIAOMI_API_KEYS"
+inject_provider_models_from_env "volcengine" "VOLCANO_ENGINE_MODELS" "VOLCANO_ENGINE_API_KEY" "VOLCANO_ENGINE_API_KEYS"
+inject_provider_models_from_env "byteplus" "BYTEPLUS_MODELS" "BYTEPLUS_API_KEY" "BYTEPLUS_API_KEYS"
+inject_provider_models_from_env "qianfan" "QIANFAN_MODELS" "QIANFAN_API_KEY" "QIANFAN_API_KEYS"
+inject_provider_models_from_env "groq" "GROQ_MODELS" "GROQ_API_KEY" "GROQ_API_KEYS"
+inject_provider_models_from_env "mistral" "MISTRAL_MODELS" "MISTRAL_API_KEY" "MISTRAL_API_KEYS"
+inject_provider_models_from_env "xai" "XAI_MODELS" "XAI_API_KEY" "XAI_API_KEYS"
+inject_provider_models_from_env "nvidia" "NVIDIA_MODELS" "NVIDIA_API_KEY" "NVIDIA_API_KEYS"
+inject_provider_models_from_env "cohere" "COHERE_MODELS" "COHERE_API_KEY" "COHERE_API_KEYS"
+inject_provider_models_from_env "together" "TOGETHER_MODELS" "TOGETHER_API_KEY" "TOGETHER_API_KEYS"
+inject_provider_models_from_env "cerebras" "CEREBRAS_MODELS" "CEREBRAS_API_KEY" "CEREBRAS_API_KEYS"
+
 # Browser configuration (managed local Chromium in HF/Docker)
 BROWSER_EXECUTABLE_PATH=""
 for candidate in /usr/bin/chromium /usr/bin/chromium-browser /snap/bin/chromium; do
@@ -540,10 +643,15 @@ fi
 # ── Trap SIGTERM for graceful shutdown ──
 graceful_shutdown() {
   echo "Shutting down..."
-  if [ -f "/home/node/app/openclaw-sync.py" ]; then
+  if [ -f "/home/node/app/openclaw-sync.py" ] && [ -n "${HF_TOKEN:-}" ]; then
     echo "Saving state before exit..."
+    timeout 8s python3 /home/node/app/openclaw-sync.py sync-once-settled || \
+      echo "Warning: could not complete settled shutdown sync"
+    sleep 1
     python3 /home/node/app/openclaw-sync.py sync-once || \
-      echo "Warning: could not complete shutdown sync"
+      echo "Warning: could not complete final shutdown sync"
+  elif [ -f "/home/node/app/openclaw-sync.py" ]; then
+    echo "HF_TOKEN not set; skipping shutdown backup sync."
   fi
   kill $(jobs -p) 2>/dev/null
   exit 0