Merge pull request #15 from anurag162008/main

feat:Prune stale dataset files and enhance sync and installation methods

.env.example

@@ -175,6 +175,13 @@ LLM_API_KEY_FALLBACK_ENABLED=true
 #      is also picked up by OpenClaw → that provider's models become
 #      available in the Control UI for manual selection.
 #   3. Rotation pools (*_API_KEYS) work for every active provider
+#
+# Optional: explicitly pin model lists per provider for Control UI visibility
+# when provider keys are configured.
+# Format: comma-separated model IDs
+# NVIDIA_MODELS=meta/llama-3.1-70b-instruct,nvidia/llama-3.1-nemotron-70b-instruct
+# OPENAI_MODELS=gpt-4o-mini,gpt-4.1
+# GROQ_MODELS=llama-3.3-70b-versatile,deepseek-r1-distill-llama-70b
 #      independently and in parallel.
 #
 # EXAMPLE — default Anthropic, also use OpenAI and Groq:

multi-provider-key-rotator.cjs

@@ -53,6 +53,7 @@ const PROVIDERS = [
     hostname:   /(?:^|\.)(?:generativelanguage\.googleapis\.com|aiplatform\.googleapis\.com)$/i,
     envPlural:  'GEMINI_API_KEYS',
     envSingular:'GEMINI_API_KEY',
+    queryParam: true,
   },
   {
     name:       'deepseek',
@@ -285,15 +286,22 @@ function patchFetch() {
       const hostname = resolveHostname(urlLike);
       const provider = matchProvider(hostname);
 
-      if (provider) {
+            if (provider) {
         const key = nextKey(provider);
         if (key) {
-          const headers        = init.headers || (input && input.headers) || undefined;
-          const patchedHeaders = setAuthHeader(headers, key);
-          init = { ...init, headers: patchedHeaders };
-
-          if (input && typeof input === 'object' && !(input instanceof URL) && input.headers) {
-            try { input = new Request(input, { headers: patchedHeaders }); } catch { /* noop */ }
+          if (provider.queryParam) {
+            // Gemini: key URL query param mein jaata hai, Bearer nahi
+            const url = new URL(typeof input === 'string' ? input : input.url);
+            url.searchParams.set('key', key);
+            input = typeof input === 'string' ? url.toString() : new Request(url.toString(), input);
+          } else {
+            const headers        = init.headers || (input && input.headers) || undefined;
+            const patchedHeaders = setAuthHeader(headers, key);
+            init = { ...init, headers: patchedHeaders };
+
+            if (input && typeof input === 'object' && !(input instanceof URL) && input.headers) {
+              try { input = new Request(input, { headers: patchedHeaders }); } catch { /* noop */ }
+            }
           }
         }
       }
@@ -319,7 +327,14 @@ function patchHttpModule(mod) {
       if (provider) {
         const key = nextKey(provider);
         if (key) {
-          if (typeof options === 'string' || options instanceof URL) {
+          if (provider.queryParam) {
+            // Gemini: ?key= query param use karo
+            const u = new URL(String(typeof options === 'string' || options instanceof URL ? options : `https://${options.hostname}${options.path || '/'}`));
+            u.searchParams.set('key', key);
+            args[0] = typeof options === 'object' && !(options instanceof URL)
+              ? { ...options, path: `${u.pathname}${u.search}` }
+              : u.toString();
+          } else if (typeof options === 'string' || options instanceof URL) {
             const u = new URL(String(options));
             args[0] = {
               protocol: u.protocol,

openclaw-sync.py

@@ -360,6 +360,30 @@ def create_snapshot_dir(source_root: Path) -> Path:
     return staging_root
 
 
+def prune_remote_deleted_files(repo_id: str, snapshot_dir: Path) -> None:
+    if HF_API is None:
+        return
+
+    local_files = {
+        path.relative_to(snapshot_dir).as_posix()
+        for path in snapshot_dir.rglob("*")
+        if path.is_file()
+    }
+
+    remote_files = HF_API.list_repo_files(repo_id=repo_id, repo_type="dataset")
+    stale_files = [
+        path for path in remote_files
+        if path not in local_files and path not in {".gitattributes"}
+    ]
+    if stale_files:
+        HF_API.delete_files(
+            delete_patterns=stale_files,
+            repo_id=repo_id,
+            repo_type="dataset",
+            commit_message="Prune stale files after workspace sync",
+        )
+
+
 def restore_workspace() -> bool:
     if not HF_TOKEN:
         write_status("disabled", "HF_TOKEN is not configured.")
@@ -459,6 +483,7 @@ def _sync_once_unlocked(
                 commit_message=f"HuggingClaw sync {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}",
                 ignore_patterns=[".git/*", ".git"],
             )
+        prune_remote_deleted_files(repo_id, snapshot_dir)
     finally:
         shutil.rmtree(snapshot_dir, ignore_errors=True)
 

start.sh

@@ -131,6 +131,57 @@ case "$LLM_PROVIDER" in
     ;;
 esac
 
+# Ensure OpenClaw provider discovery can see per-provider keys even when users
+# configure only *_API_KEYS pools. Mirror first pool key into singular env.
+promote_first_pool_key() {
+  local singular_var="$1"
+  local pool_var="$2"
+  local singular_val="${!singular_var:-}"
+  local pool_val="${!pool_var:-}"
+
+  [ -n "$singular_val" ] && return 0
+  [ -n "$pool_val" ] || return 0
+
+  local first
+  first=$(printf '%s' "$pool_val" | tr ',' '\n' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | awk 'NF{print; exit}')
+  [ -n "$first" ] || return 0
+  export "${singular_var}=$first"
+}
+
+promote_first_pool_key "ANTHROPIC_API_KEY" "ANTHROPIC_API_KEYS"
+promote_first_pool_key "OPENAI_API_KEY" "OPENAI_API_KEYS"
+promote_first_pool_key "GEMINI_API_KEY" "GEMINI_API_KEYS"
+promote_first_pool_key "GEMINI_API_KEY" "GOOGLE_API_KEYS"
+promote_first_pool_key "DEEPSEEK_API_KEY" "DEEPSEEK_API_KEYS"
+promote_first_pool_key "OPENROUTER_API_KEY" "OPENROUTER_API_KEYS"
+promote_first_pool_key "KILOCODE_API_KEY" "KILOCODE_API_KEYS"
+promote_first_pool_key "OPENCODE_API_KEY" "OPENCODE_API_KEYS"
+promote_first_pool_key "ZAI_API_KEY" "ZAI_API_KEYS"
+promote_first_pool_key "MOONSHOT_API_KEY" "MOONSHOT_API_KEYS"
+promote_first_pool_key "MINIMAX_API_KEY" "MINIMAX_API_KEYS"
+promote_first_pool_key "XIAOMI_API_KEY" "XIAOMI_API_KEYS"
+promote_first_pool_key "VOLCANO_ENGINE_API_KEY" "VOLCANO_ENGINE_API_KEYS"
+promote_first_pool_key "BYTEPLUS_API_KEY" "BYTEPLUS_API_KEYS"
+promote_first_pool_key "QIANFAN_API_KEY" "QIANFAN_API_KEYS"
+promote_first_pool_key "MODELSTUDIO_API_KEY" "MODELSTUDIO_API_KEYS"
+promote_first_pool_key "KIMI_API_KEY" "KIMI_API_KEYS"
+promote_first_pool_key "MISTRAL_API_KEY" "MISTRAL_API_KEYS"
+promote_first_pool_key "XAI_API_KEY" "XAI_API_KEYS"
+promote_first_pool_key "NVIDIA_API_KEY" "NVIDIA_API_KEYS"
+promote_first_pool_key "GROQ_API_KEY" "GROQ_API_KEYS"
+promote_first_pool_key "COHERE_API_KEY" "COHERE_API_KEYS"
+promote_first_pool_key "TOGETHER_API_KEY" "TOGETHER_API_KEYS"
+promote_first_pool_key "CEREBRAS_API_KEY" "CEREBRAS_API_KEYS"
+promote_first_pool_key "VENICE_API_KEY" "VENICE_API_KEYS"
+promote_first_pool_key "SYNTHETIC_API_KEY" "SYNTHETIC_API_KEYS"
+promote_first_pool_key "COPILOT_GITHUB_TOKEN" "COPILOT_GITHUB_TOKENS"
+promote_first_pool_key "HUGGINGFACE_HUB_TOKEN" "HUGGINGFACE_HUB_TOKENS"
+
+# Compatibility aliases for Google provider secrets some users already have.
+if [ -z "${GEMINI_API_KEY:-}" ] && [ -n "${GOOGLE_API_KEY:-}" ]; then
+  export GEMINI_API_KEY="$GOOGLE_API_KEY"
+fi
+
 # ── Setup directories ──
 mkdir -p /home/node/.openclaw/agents/main/sessions
 mkdir -p /home/node/.openclaw/credentials
@@ -285,6 +336,82 @@ if [ -n "$CUSTOM_PROVIDER_NAME" ] || [ -n "$CUSTOM_BASE_URL" ] || [ -n "$CUSTOM_
   fi
 fi
 
+# Optional: explicitly expose provider model lists in Control UI when
+# provider keys are configured. Format:
+#   NVIDIA_MODELS=model1,model2
+#   OPENAI_MODELS=gpt-4o-mini,gpt-4.1
+# This helps when provider auto-discovery does not populate models reliably.
+inject_provider_models_from_env() {
+  local provider="$1"
+  local models_env="$2"
+  local key_env_single="$3"
+  local key_env_pool="$4"
+  local models_csv="${!models_env:-}"
+  local single_key="${!key_env_single:-}"
+  local pool_keys="${!key_env_pool:-}"
+
+  # Only inject when both:
+  # 1) provider has at least one configured key
+  # 2) explicit model list env is provided
+  if [ -z "$models_csv" ] || { [ -z "$single_key" ] && [ -z "$pool_keys" ]; }; then
+    return 0
+  fi
+
+  local models_json
+  models_json=$(printf '%s' "$models_csv" \
+    | tr ',' '\n' \
+    | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' \
+    | awk 'NF' \
+    | jq -R . \
+    | jq -s 'map({id: ., name: .}) | unique_by(.id)')
+
+  CONFIG_JSON=$(jq \
+    --arg provider "$provider" \
+    --argjson models "$models_json" \
+    '.models.mode = "merge"
+     | .models.providers[$provider] = ((.models.providers[$provider] // {}) + {models: $models})' <<<"$CONFIG_JSON")
+}
+
+# Built-in provider model envs (optional)
+inject_provider_models_from_env "anthropic" "ANTHROPIC_MODELS" "ANTHROPIC_API_KEY" "ANTHROPIC_API_KEYS"
+inject_provider_models_from_env "openai" "OPENAI_MODELS" "OPENAI_API_KEY" "OPENAI_API_KEYS"
+inject_provider_models_from_env "openai-codex" "OPENAI_MODELS" "OPENAI_API_KEY" "OPENAI_API_KEYS"
+inject_provider_models_from_env "google" "GEMINI_MODELS" "GEMINI_API_KEY" "GEMINI_API_KEYS"
+inject_provider_models_from_env "google-vertex" "GEMINI_MODELS" "GEMINI_API_KEY" "GEMINI_API_KEYS"
+inject_provider_models_from_env "deepseek" "DEEPSEEK_MODELS" "DEEPSEEK_API_KEY" "DEEPSEEK_API_KEYS"
+inject_provider_models_from_env "openrouter" "OPENROUTER_MODELS" "OPENROUTER_API_KEY" "OPENROUTER_API_KEYS"
+inject_provider_models_from_env "kilocode" "KILOCODE_MODELS" "KILOCODE_API_KEY" "KILOCODE_API_KEYS"
+inject_provider_models_from_env "opencode" "OPENCODE_MODELS" "OPENCODE_API_KEY" "OPENCODE_API_KEYS"
+inject_provider_models_from_env "opencode-go" "OPENCODE_MODELS" "OPENCODE_API_KEY" "OPENCODE_API_KEYS"
+inject_provider_models_from_env "zai" "ZAI_MODELS" "ZAI_API_KEY" "ZAI_API_KEYS"
+inject_provider_models_from_env "z-ai" "ZAI_MODELS" "ZAI_API_KEY" "ZAI_API_KEYS"
+inject_provider_models_from_env "z.ai" "ZAI_MODELS" "ZAI_API_KEY" "ZAI_API_KEYS"
+inject_provider_models_from_env "zhipu" "ZAI_MODELS" "ZAI_API_KEY" "ZAI_API_KEYS"
+inject_provider_models_from_env "moonshot" "MOONSHOT_MODELS" "MOONSHOT_API_KEY" "MOONSHOT_API_KEYS"
+inject_provider_models_from_env "kimi-coding" "KIMI_MODELS" "KIMI_API_KEY" "KIMI_API_KEYS"
+inject_provider_models_from_env "minimax" "MINIMAX_MODELS" "MINIMAX_API_KEY" "MINIMAX_API_KEYS"
+inject_provider_models_from_env "modelstudio" "MODELSTUDIO_MODELS" "MODELSTUDIO_API_KEY" "MODELSTUDIO_API_KEYS"
+inject_provider_models_from_env "qwen" "MODELSTUDIO_MODELS" "MODELSTUDIO_API_KEY" "MODELSTUDIO_API_KEYS"
+inject_provider_models_from_env "xiaomi" "XIAOMI_MODELS" "XIAOMI_API_KEY" "XIAOMI_API_KEYS"
+inject_provider_models_from_env "volcengine" "VOLCANO_ENGINE_MODELS" "VOLCANO_ENGINE_API_KEY" "VOLCANO_ENGINE_API_KEYS"
+inject_provider_models_from_env "volcengine-plan" "VOLCANO_ENGINE_MODELS" "VOLCANO_ENGINE_API_KEY" "VOLCANO_ENGINE_API_KEYS"
+inject_provider_models_from_env "byteplus" "BYTEPLUS_MODELS" "BYTEPLUS_API_KEY" "BYTEPLUS_API_KEYS"
+inject_provider_models_from_env "byteplus-plan" "BYTEPLUS_MODELS" "BYTEPLUS_API_KEY" "BYTEPLUS_API_KEYS"
+inject_provider_models_from_env "qianfan" "QIANFAN_MODELS" "QIANFAN_API_KEY" "QIANFAN_API_KEYS"
+inject_provider_models_from_env "groq" "GROQ_MODELS" "GROQ_API_KEY" "GROQ_API_KEYS"
+inject_provider_models_from_env "mistral" "MISTRAL_MODELS" "MISTRAL_API_KEY" "MISTRAL_API_KEYS"
+inject_provider_models_from_env "mistralai" "MISTRAL_MODELS" "MISTRAL_API_KEY" "MISTRAL_API_KEYS"
+inject_provider_models_from_env "xai" "XAI_MODELS" "XAI_API_KEY" "XAI_API_KEYS"
+inject_provider_models_from_env "x-ai" "XAI_MODELS" "XAI_API_KEY" "XAI_API_KEYS"
+inject_provider_models_from_env "nvidia" "NVIDIA_MODELS" "NVIDIA_API_KEY" "NVIDIA_API_KEYS"
+inject_provider_models_from_env "cohere" "COHERE_MODELS" "COHERE_API_KEY" "COHERE_API_KEYS"
+inject_provider_models_from_env "together" "TOGETHER_MODELS" "TOGETHER_API_KEY" "TOGETHER_API_KEYS"
+inject_provider_models_from_env "cerebras" "CEREBRAS_MODELS" "CEREBRAS_API_KEY" "CEREBRAS_API_KEYS"
+inject_provider_models_from_env "huggingface" "HUGGINGFACE_MODELS" "HUGGINGFACE_HUB_TOKEN" "HUGGINGFACE_HUB_TOKENS"
+inject_provider_models_from_env "venice" "VENICE_MODELS" "VENICE_API_KEY" "VENICE_API_KEYS"
+inject_provider_models_from_env "synthetic" "SYNTHETIC_MODELS" "SYNTHETIC_API_KEY" "SYNTHETIC_API_KEYS"
+inject_provider_models_from_env "github-copilot" "GITHUB_COPILOT_MODELS" "COPILOT_GITHUB_TOKEN" "COPILOT_GITHUB_TOKENS"
+
 # Browser configuration (managed local Chromium in HF/Docker)
 BROWSER_EXECUTABLE_PATH=""
 for candidate in /usr/bin/chromium /usr/bin/chromium-browser /snap/bin/chromium; do