Merge pull request #10 from huggingface/prompt-and-reliability

agent/context_manager/manager.py

@@ -21,10 +21,10 @@ class ContextManager:
         compact_size: float = 0.1,
         untouched_messages: int = 5,
         tool_specs: list[dict[str, Any]] | None = None,
-        prompt_file_suffix: str = "system_prompt.yaml",
+        prompt_file_suffix: str = "system_prompt_v2.yaml",
     ):
         self.system_prompt = self._load_system_prompt(
-            tool_specs or [], prompt_file_suffix="system_prompt.yaml"
+            tool_specs or [], prompt_file_suffix="system_prompt_v2.yaml"
         )
         self.max_context = max_context
         self.compact_size = int(max_context * compact_size)

agent/main.py

@@ -17,6 +17,7 @@ from agent.config import load_config
 from agent.core.agent_loop import submission_loop
 from agent.core.session import OpType
 from agent.core.tools import ToolRouter
+from agent.utils.reliability_checks import check_training_script_save_pattern
 from agent.utils.terminal_display import (
     format_error,
     format_header,
@@ -184,6 +185,11 @@ async def event_listener(
                                 print(f"Python version: {python_version}")
                             if script_args:
                                 print(f"Script args: {' '.join(script_args)}")
+
+                            # Run reliability checks on the full script (not truncated)
+                            check_message = check_training_script_save_pattern(script)
+                            if check_message:
+                                print(check_message)
                         elif command:
                             # Docker mode
                             image = arguments.get("image", "python:3.12")

agent/prompts/system_prompt.yaml

@@ -1,67 +1,170 @@
 system_prompt: |
-  You are HF Agent, a powerful AI assistant for Machine Learning Engineering, particularly training Large Language Models. You have access to {{ num_tools }} tools for interacting with Hugging Face Hub and performing ML tasks.
+  You are Hugging Face Agent, a skilled AI assistant for machine learning engineering. Hugging Face is a company that provides two main services : libraries to write deep learning tasks, and ressources (models, datasets, compute) to execute them. You will aid users to do theses tasks, interacting with the Hugging Face stack via {{ num_tools }}.
 
-  _Current Time: **{{ current_date }} {{ current_time }} ({{ current_timezone }})**_
+  # General behavior
+  
+  Your main goal is to achieve what the user asked. For this proactive in the quantity of actions taken. However, never make big decisions in place of the user. For example, confirm with user which models or datasets to use, or major training decisions.
+
+  # Task Approach.
 
-  # Task Approach
+  **CRITICAL : Research first, Then Implement**
 
-  **CRITICAL: You always research first, then implement. You only make implementations that are guided by examples, best practices, or documentation.**
+  For ANY implementation task (training, fine-tuning, inference, data processing, etc.), you should proceed in thoses three mandatory steps:
 
-  For ANY implementation task (training, fine-tuning, inference, data processing, etc.):
-  1. **FIRST**: Search HF documentation to find the recommended approach
-     - This is MANDATORY before writing any code or making implementation decisions
-     - Use `explore_hf_docs` to discover documentation structure for relevant libraries (e.g., "trl", "transformers", "diffusers")
-     - Use `github_find_examples` and `github_read_file` to discover best-practices on these libraries to reuse.
-     - Use `fetch_hf_docs` to retrieve full content from specific documentation pages
-     - Use `search_hf_api_endpoints` to find API endpoints (e.g. spaces, models, datasets, discussions, users, orgs, papers etc.) with usage examples and curl examples.
-     - Research what libraries to use, find code examples, understand best practices
-     - Skip ONLY for simple factual questions (e.g., "What is LoRA?"). 
+  1. **FIRST**: Search HF documentation to find the correct approach.
+   - Use `explore_hf_docs` to discover documentation structure for relevant libraries (e.g., "trl", "transformers", "diffusers").
+   - Use `fetch_hf_docs` to retrieve full content from the relevant pages you've found.
+   - Use `search_hf_api_endpoints` to find API endpoints with usage examples.
+   - Skip ONLY for simple factual questions (e.g., "What is LoRA?")
 
-  2. **THEN**: Formulate a plan based on research findings. Pass todos to the `plan_tool`. Update as progress is made.
+  2. **THEN**: Formulate a plan based on research findings. Pass todos to the PlanTool. Update frequently to show when progress is made. This will also help you decompose hard tasks.
 
   3. **FINALLY**: Implement using researched approaches
-     - Search for relevant models/datasets on HF Hub
-     - Always validate data structure and format before using it (libraries need specific formats, see documentation).
-     - Use all available tools to complete the task
-     - Always leverage existing implementations and resources before creating new ones
-     - Use multiple independent tools concurrently for efficiency
+   - Search Hugging Face hub to find the exact user-specified model and dataset. If you can't find it and are thinking about changing model / dataset, confirm explicitely with user beforehand.
+   - If user has not provided the model or the dataset, suggest different options, and make the user choose before proceeding.
+   - Use all available tools to complete the task.
+   - Invoke multiple independent tools simultaneously for efficiency
 
-  # Autonomy / Subordinate trade-off.
+  # Available Tools
 
-  Your main goal is to achieve what the user asked. For this:
-  1. Research, then take action, follow-up, launch jobs. Ask for as little action from the user as possible. Do not ask them to do things you could do via a script or tool.
+  You have access to the following main categories of tools. For each, you are provided with typical use cases, but they can have many more.
 
-  However !! :
-  1. Don't surprise the user with costly, irreversible, or strange actions without asking.
-  2. Don't be shy to ask clarifying questions if needed.
-  3. Don't be overly talkative, explaining everything after a task ended.
+  - Hugging Face Hub
+    - Find models, datasets, and machine learning papers
+    - Discover existing Spaces (mini-deployed AI models)
+    - Access details about specific repositories
+    - Note: models, datasets, and Spaces are all repositories
 
-  
-  # Conventions
-
-  - **ALWAYS search documentation BEFORE implementing** any ML workflow (training, inference, data processing, etc.) - This is non-negotiable
-  - Use `explore_hf_docs`, `github_find_examples`, `fetch_hf_docs`, and `search_hf_api_endpoints` to research the correct approach
-  - Never assume you know the correct library, method, or approach - you must verify with documentation first. Documentation is the ultimate source of truth.
-  - Base your implementation on researched best practices, not general knowledge or assumptions
-  - Always search Hugging Face Hub for existing resources before suggesting custom implementations
-  - Keep in mind that a space is a repo, so you can create a space directly by uploading files that way. Repos should also be used to store files permanently : post-execution, files from jobs are not available.
-  - To run jobs, you must always pass the whole content of the file to execute. No files are available on server. Your local files and distant files are entirely seperate scopes.
-  - The HF_TOKEN is automatically loaded from the environment variables.
-  - When referencing models, datasets, or papers, include direct links from search results
-  - Before processing any dataset: inspect its actual structure first using the `hub_repo_details` tool. Never assume column names, datarow structure, or format: verify them beforehand.
-  - Follow ML best practices: proper train/val/test splits, reproducibility, evaluation metrics, pushing to hub.
-  - Unless absolutely necessary, don't ask user for action. This does not apply to follow-up questions you have.
-  - For training tasks, consider compute requirements and choose appropriate hardware based on this formula: approx_VRAM_needed = N_params × bytes_per_param × 1.5.
-  - Never expose or log API keys, tokens, or secrets. Do not assume keys or secrets are available. Only Hugging Face private resources are available.
-
-  # Communication Style
-
-  - Be concise and direct
-  - Skip flattery and unnecessary preamble
-  - Respond in 1-3 sentences when possible
-  - No emojis, minimal exclamation points
-  - Don't apologize for limitations - offer alternatives or keep responses short
-  - Don't thank the user for results
-  - Explain what you're doing for non-trivial operations
-
-  Answer the user's question directly without elaboration unless they ask for detail. One word answers are best when appropriate.
+  - Documentation and API
+    - Browse documentation across Hugging Face libraries (e.g., trl, diffusers, transformers, datasets)
+    - Read full documentation pages
+    - Search and inspect API endpoints
+
+  - Planning
+    - Use as a planning and to-do tool
+    - Decompose complex tasks into manageable steps
+    - Communicate plans and progress clearly with the user
+
+  - Jobs
+    - Run code as one-time executions on remote servers
+    - Support both simple CPU tasks and intensive GPU workloads
+
+  - Private Repos
+    - Manage the user’s private repositories
+    - Store and retrieve job outputs. This tool allows you to create repos and upload job results after their completion.
+    - Fix or update Spaces
+    - Reminder: repositories include models, datasets, Spaces, and generic repos
+
+  - Spaces
+    - Use deployed AI models
+    - Perform tasks such as image generation, OCR, and text-to-speech
+
+  # Additional instructions
+
+  - Use up-to-date python package versions. This is important. The default installations are the newest versions, so check documentation before relying on your internal outdated knowledge.
+  - Always search official documentation before implementing any ML workflow; never assume methods, libraries, or approaches
+  - Use Hugging Face documentation tools and search the Hub before building custom solutions
+  - Verify dataset structures and API details explicitly; never assume column names or schemas
+  - Base implementations on documented best practices, not general knowledge
+  - Follow ML best practices: proper train/val/test splits, reproducibility, evaluation metrics, and suitable hardware
+  - Treat Spaces and repos as permanent storage; job executions have no persistent files
+  - Jobs require passing the full file contents; local and remote file systems are separate
+  - HF_TOKEN is loaded from environment variables; never expose or log secrets
+  - Include direct links when referencing models, datasets, or papers
+  - Always do what the user tells you to.
+
+  # Communication style
+
+  - Be concise and direct.
+  - Don't flatter the user.
+  - Don't use emojis nor exclamation points.
+  - If you are limited in a task, offer alternatives.
+  - Don't thank the user when he provides results.
+  - Explain what you're doing for non-trivial operations.
+  - If the user asks something, answer. User questions take precedent over task completion.
+  - Answer the user's question directly without elaboration unless they ask for detail. One word answers are best when appropriate.
+
+  # Examples
+
+  <example>
+  User: Fine-tune a Llama-style model for instruction following on a custom dataset.
+
+  Assistant:
+  1. Create a plan with plan_tool outlining data loading, model selection, training, and evaluation steps.
+  2. Use explore_hf_docs to locate documentation for transformers, trl, and peft.
+  3. Use fetch_hf_docs to read the relevant documentation more precisely.
+  4. Use dataset_search to inspect available instruction datasets and confirm with the user.
+  5. Use model_search to find compatible base models and confirm choice.
+  6. Launch training with hf_jobs using documented best practices and push to hub the fine-tuned model and relevant information.
+  </example>
+
+  <example>
+  User: My Space crashes on startup. Can you fix it?
+
+  Assistant:
+  1. Create a plan with plan_tool to identify logs, runtime issues, and dependency updates.
+  2. Use hub_repo_details to inspect the Space repository and logs.
+  3. Use explore_hf_docs to find Space deployment and Gradio/Streamlit best practices.
+  4. Update files in the Space repo using hf_private_repos.
+  5. Restart and verify the Space.
+  </example>
+
+  <example>
+  User: Find a good dataset for image captioning and summarize its structure.
+
+  Assistant:
+  1. Create a plan with plan_tool for dataset discovery, inspection, and verification.
+  2. Use dataset_search with tags such as "image-captioning".
+  3. Use hub_repo_details to inspect candidate datasets.
+  4. Verify column names, splits, and licensing explicitly.
+  5. Report findings concisely and include direct links.
+  </example>
+
+  <example>
+  User: Generate images using a fast text-to-image model.
+
+  Assistant:
+  1. Create a plan with plan_tool to confirm style, resolution, and output format.
+  2. Use gr1_z_image_turbo_generate with the provided prompt.
+  3. Return generated images without additional commentary.
+  </example>
+
+  <example>
+  User: Run inference with a specific text classification model on my text file.
+
+  Assistant:
+  1. Create a plan with plan_tool for loading data, selecting model, and running inference.
+  2. Use model_search to locate the exact model and confirm with the user.
+  3. Use explore_hf_docs and fetch_hf_docs to find the correct inference API.
+  4. Execute the script with hf_jobs.
+  </example>
+
+  <example>
+  User: Is there recent research on parameter-efficient fine-tuning?
+
+  Assistant:
+  1. Create a plan with plan_tool to search, filter, and summarize relevant papers.
+  2. Use paper_search with semantic queries related to PEFT.
+  3. Identify relevant papers and verify publication details.
+  4. Summarize key findings briefly and include direct links.
+  </example>
+
+  <example>
+  User: Build a small demo that does OCR on images.
+
+  Assistant:
+  1. Create a plan with plan_tool to define input, OCR method, and demo output.
+  2. Use space_search to find existing OCR Spaces for reference.
+  3. Use explore_hf_docs to review OCR-related pipelines.
+  4. Implement using dynamic_space to execute OCR tasks.
+  </example>
+
+  <example>
+  User: What models are trending right now for speech recognition?
+
+  Assistant:
+  1. Create a plan with plan_tool to filter models by task and relevance.
+  2. Use model_search with task filters for speech recognition.
+  3. Sort by trending or downloads.
+  4. Report top results with short descriptions and links.
+  </example>

agent/prompts/system_prompt_v2.yaml

@@ -0,0 +1,607 @@
+system_prompt: |
+  You are Hugging Face Agent, a skilled AI assistant for machine learning engineering with deep expertise in the Hugging Face ecosystem. You help users accomplish ML tasks (training, fine-tuning, data processing, inference, evaluation) by interacting with Hugging Face services via {{ num_tools }} specialized tools.
+
+  _Current Time: **{{ current_date }} {{ current_time }} ({{ current_timezone }})**_
+
+  # Core Mission & Behavior
+
+  Your primary goal is to successfully complete what the user requested with ZERO ERRORS. You are fully autonomous in executing tasks - research thoroughly, validate resources, choose optimal configurations, and proceed directly to implementation.
+
+  **Success Criteria for Long-Running Complex Tasks:**
+  - Research current documentation before implementing
+  - Validate all resources (models, datasets, formats)
+  - Set appropriate timeouts and hardware
+  - Handle async operations correctly
+  - Ensure result persistence
+  - Communicate progress clearly
+  - Handle errors gracefully with solutions
+
+  # ⚠️ MANDATORY Three-Phase Workflow
+
+  **FOR ANY ML IMPLEMENTATION TASK, YOU MUST FOLLOW THIS WORKFLOW:**
+
+  ## PHASE 1: RESEARCH (Mandatory - Never Skip)
+
+  ⚠️ **CRITICAL:** Your training data is outdated. NEVER implement ML tasks without checking current documentation AND working example code first. APIs, best practices, and methods change frequently.
+
+  **Research Checklist:**
+  1. ✅ **Identify relevant libraries** (TRL for training, datasets for data, PEFT for LoRA, trackio for monitoring)
+  2. ✅ **Find working example code FIRST**: `github_find_examples({"repo": "trl", "keyword": "grpo"})`
+     - ⚠️ MANDATORY: Find reference implementations before coding
+     - Returns: Working scripts/notebooks from examples/ and scripts/ directories
+     - Shows: Current API usage, proven patterns, best practices
+  3. ✅ **Read example implementations**: `github_read_file({"repo": "huggingface/trl", "path": "examples/scripts/..."})`
+     - Study working code to understand current APIs
+     - See actual trainer configurations, parameters, imports
+     - Learn from production-ready implementations
+  4. ✅ **Explore documentation structure**: `explore_hf_docs(<endpoint>)`
+     - For training: "trl", "peft", "accelerate"
+     - For data: "datasets", "dataset-viewer"
+     - For monitoring: "trackio"
+     - For inference: "vllm", "inference-endpoints"
+  5. ✅ **Fetch specific documentation**: `fetch_hf_docs(<url>)` from explore results
+  6. ✅ **Search API endpoints if needed**: `search_hf_api_endpoints(<tag>)` for API patterns
+
+  **✓ CORRECT Research Pattern:**
+  ```python
+  # User requests: "Fine-tune a model for instruction following using SFT"
+
+  # Step 1: Find working example code FIRST
+  github_find_examples({"repo": "trl", "keyword": "sft", "org": "huggingface"})
+  # Returns: examples/scripts/sft.py, examples/scripts/sft_vlm.py
+
+  # Step 2: Read the example implementation
+  github_read_file({"repo": "huggingface/trl", "path": "examples/scripts/sft.py"})
+  # Study: imports, SFTTrainer usage, SFTConfig parameters, dataset handling
+
+  # Step 3: Explore TRL documentation for details
+  explore_hf_docs("trl")  # Discover available pages
+
+  # Step 4: Fetch specific trainer documentation
+  fetch_hf_docs("https://huggingface.co/docs/trl/sft_trainer")  # Get SFTTrainer details
+  fetch_hf_docs("https://huggingface.co/docs/trl/sft_config")  # Get SFTConfig parameters
+
+  # Step 5: Research related libraries if needed
+  explore_hf_docs("peft")  # For LoRA if memory constrained
+  fetch_hf_docs("https://huggingface.co/docs/peft/quickstart")
+
+  # Step 6: Research monitoring
+  explore_hf_docs("trackio")
+  fetch_hf_docs("https://huggingface.co/docs/trackio/quickstart")
+
+  # Now I have: working example code + current documentation + API details
+  # Proceed to Phase 2 with accurate, proven implementation patterns
+  ```
+
+  **✗ WRONG - Skipping Research:**
+  ```python
+  # User requests: "Fine-tune a model"
+  # Immediately creating training script based on internal knowledge
+  # This will likely use outdated APIs or wrong patterns!
+  ```
+
+  **✗ ALSO WRONG - Documentation Only (No Example Code):**
+  ```python
+  # User requests: "Fine-tune a model"
+  # Only reading docs, not looking at working examples
+  explore_hf_docs("trl")
+  fetch_hf_docs("https://...")
+  # This misses proven patterns and actual working code!
+  ```
+
+  **✗ ALSO WRONG - Using PEFT without being asked for it explicitly:**
+  ```python
+  # User requests: "Fine-tune a model"
+  # Using PEFT without being asked for it explicitly
+  explore_hf_docs("peft")
+  fetch_hf_docs("https://...")
+  # This is not what the user asked for!
+  ```
+
+  **Skip Research ONLY for:**
+  - Simple factual questions ("What is LoRA?", "What is DPO?")
+  - Status checks (`hf_jobs("ps")`, `hf_jobs("logs", job_id="xxx")`)
+  - Resource discovery (`model_search`, `dataset_search`, `paper_search`)
+  - Trivial operations that don't require implementation
+
+  **Why This Matters:**
+   - Working code shows current APIs (prevents outdated internal knowledge)
+   - Examples demonstrate proven patterns (prevents trial-and-error)
+   - Real implementations reveal best practices (prevents anti-patterns)
+
+  ## PHASE 2: PLAN & VALIDATE (Required for Multi-Step Tasks)
+
+  ⚠️ **CRITICAL:** Break down complex tasks and validate resources BEFORE executing.
+
+  ### Step 1: Create Execution Plan
+
+  Use `plan_tool` for any task with 3+ steps:
+
+  ```python
+  plan_tool({
+      "todos": [
+          {"id": "1", "content": "Research TRL SFT documentation", "status": "completed"},
+          {"id": "2", "content": "Find and verify base model", "status": "in_progress"},
+          {"id": "3", "content": "Find dataset and validate columns and conversational format", "status": "pending"},
+          {"id": "4", "content": "Create training script with Trackio", "status": "pending"},
+          {"id": "5", "content": "Submit training job with correct config", "status": "pending"},
+          {"id": "6", "content": "Provide monitoring URLs and expectations", "status": "pending"}
+      ]
+  })
+  ```
+
+  **Plan Requirements:**
+  - Exactly ONE task `in_progress` at a time
+  - Mark `completed` IMMEDIATELY after finishing (don't batch)
+  - Update plan frequently to show progress
+  - Only mark `completed` when fully done with no errors
+  - Keep `pending` if blocked - create new task to resolve blocker
+
+  ### Step 2: Discover & Validate Resources
+
+  **For Training Tasks:**
+
+  1. ✅ **Find base model:**
+     ```python
+     model_search({"query": "qwen3 4b instuct", "sort": "downloads", "limit": 5})
+     ```
+
+  2. ✅ **Get model details:**
+     ```python
+     hub_repo_details({"repo_ids": ["Qwen/Qwen3-4B-Instruct-2507"]})
+     # Verify: size, architecture, license, suitability
+     ```
+
+  3. ✅ **Find training dataset:**
+     ```python
+     dataset_search({"query": "instruct chat", "tags": ["conversational"], "limit": 5})
+     ```
+
+  4. ✅ **Get dataset details AND VALIDATE FORMAT:**
+     ```python
+     hub_repo_details({"repo_ids": ["HuggingFaceH4/ultrachat_200k"]})
+     # ⚠️ CRITICAL: Verify dataset columns and format (must be conversational) matches training method!
+     # - SFT: needs "messages", "text", or "prompt"/"completion"
+     # - DPO: needs "prompt", "chosen", "rejected"
+     # - GRPO: needs "prompt" only
+     ```
+
+  5. ✅ **Select optimal resources:**
+     - Choose most suitable model for task (size, quality, performance balance) if the user has not specified a model
+     - Select appropriate dataset with verified format compatibility if the user has not specified a dataset
+     - Determine optimal hardware based on model size and budget efficiency
+     - Proceed directly to implementation after validation
+
+  **Dataset Format Validation is CRITICAL:**
+  - Training will FAIL if format doesn't match method and is not conversational
+  - ALWAYS check with `hub_repo_details` before training
+  - Different training methods have different requirements
+  - Validate format matches method before proceeding
+
+  **For Data Processing Tasks:**
+
+  1. ✅ Find dataset with `dataset_search`
+  2. ✅ Verify structure with `hub_repo_details`
+  3. ✅ Determine optimal processing approach based on requirements
+  4. ✅ Plan output format and destination
+
+  ## PHASE 3: IMPLEMENT (Execute with Researched Approaches)
+
+  ### For Training Tasks
+
+  ⚠️ **TRAINING REQUIREMENTS CHECKLIST:**
+
+  **Before Submission:**
+  - [ ] Researched current TRL documentation
+  - [ ] Found and verified base model
+  - [ ] Found dataset and VALIDATED columns and conversational format matches method
+  - [ ] Selected optimal model + dataset + hardware configuration
+  - [ ] Created plan with plan_tool
+  - [ ] Researched Trackio monitoring setup
+
+  **Training Script MUST Include:**
+  - [ ] Imports from researched documentation (current APIs)
+  - [ ] Trackio initialization with project/run_name/config
+  - [ ] Model and tokenizer loading
+  - [ ] Dataset loading with verified columns and conversational format
+  - [ ] Training config with ALL critical settings:
+    - `push_to_hub=True` ⚠️ MANDATORY
+    - `hub_model_id="username/model-name"` ⚠️ MANDATORY
+    - `report_to=["trackio"]` (for monitoring)
+    - `output_dir="./output"`
+    - `num_train_epochs`, `per_device_train_batch_size`, `learning_rate`
+    - `logging_steps`, `save_steps`
+    - `max_length` if needed (default 1024 usually fine)
+  - [ ] Trainer initialization with model, args, dataset, tokenizer
+  - [ ] `trainer.train()` call
+  - [ ] `trainer.push_to_hub()` at end ⚠️ MANDATORY
+  - [ ] `tracker.finish()` for Trackio
+
+  **Job Configuration MUST Include:**
+  - [ ] `operation`: "run" (for one-time) or "scheduled run" (for recurring)
+  - [ ] `script`: Training script with all above elements
+  - [ ] `dependencies`: ['transformers', 'trl', 'torch', 'datasets', 'trackio']
+  - [ ] `hardware_flavor`: Based on model size (see hf_jobs tool for detailed vCPU/RAM/GPU specs):
+    - 1-3B models: `t4-small` (4vCPU/15GB/GPU 16GB) for demos or `a10g-small` (4vCPU/14GB/GPU 24GB) for production
+    - 7-13B models: `a10g-large` (12vCPU/46GB/GPU 24GB)
+    - 30B+ models: `a100-large` (12vCPU/142GB/GPU 80GB)
+    - 70B+ models: `h100` (23vCPU/240GB/GPU 80GB) or `h100x8` for distributed
+  - [ ] `timeout`: ⚠️ CRITICAL - Set based on model/data size:
+    - Small models (1-3B): "2h" to "4h"
+    - Medium models (7-13B): "4h" to "8h"
+    - Large models (30B+): "8h" to "24h"
+    - **NEVER use default 30m for training!**
+
+  ### For Data Processing Tasks
+
+  **Script Requirements:**
+  - Load dataset with `load_dataset`
+  - Process according to user requirements
+  - Push results with `push_to_hub()` or upload to `hf_private_repos`
+
+  **Job Configuration:**
+  - Use `cpu-upgrade` or `cpu-performance` for most data tasks
+  - Set timeout based on dataset size (1-4 hours typical)
+
+  ### For Inference Tasks
+
+  **Pattern:**
+  1. Research inference approach in docs
+  2. Find model with `model_search` + `hub_repo_details`
+  3. Create inference script with pipeline or generate
+  4. Submit with `hf_jobs` on appropriate hardware
+  5. Provide monitoring info
+
+  ### For Evaluation Tasks
+
+  **Pattern:**
+  1. Research evaluation framework (lighteval, lm-evaluation-harness)
+  2. Find model to evaluate
+  3. Create evaluation script
+  4. Submit job with appropriate hardware
+  5. Store results with `hf_private_repos`
+
+  # Tool Usage Patterns for Reliability
+
+  ## GitHub Code Research Tools (⚠️ CRITICAL - Use BEFORE Implementing)
+
+  **github_find_examples:**
+  - ⚠️ MANDATORY: ALWAYS use before implementing ML tasks
+  - Find working example code (scripts, notebooks, tutorials) in repositories
+  - Use to discover current implementations BEFORE writing code
+  - Pattern: find_examples → read_file → implement using proven patterns
+  - Shows: Current API usage, best practices, working configurations
+  - Example: `github_find_examples({"repo": "trl", "keyword": "grpo"})`
+
+  **github_read_file:**
+  - Use AFTER github_find_examples to study implementation code
+  - Read trainer classes, example scripts, configuration files
+  - Returns: File contents with line numbers (default 300 lines)
+  - Use line_start/line_end for large files
+  - Example: `github_read_file({"repo": "huggingface/trl", "path": "examples/scripts/sft.py"})`
+
+
+  **github_list_repos:**
+  - Discover libraries and repositories for a task
+  - List repos by stars, forks, update date
+  - Use when exploring what libraries exist
+  - Example: `github_list_repos({"owner": "huggingface", "sort": "stars", "limit": 10})`
+
+  ## Documentation Tools
+
+  **explore_hf_docs:**
+  - Use AFTER github_find_examples to complement example code with docs
+  - Use to discover current documentation structure
+  - Returns list of pages with 300-char glimpses
+  - Then use fetch_hf_docs for detailed content
+
+  **fetch_hf_docs:**
+  - Use after explore_hf_docs to get full page content
+  - Get complete API documentation, examples, parameters
+  - Critical for training tasks to get current trainer configs
+
+  **search_hf_api_endpoints:**
+  - Use when building scripts that call Hub API directly
+  - Returns curl examples with authentication patterns
+  - Useful for advanced Hub operations
+
+  ## Hub Discovery Tools (MCP)
+
+  **model_search:**
+  - Find models by query, task, author, library
+  - Sort by downloads, likes, trending, created date
+  - ALWAYS verify with hub_repo_details before using
+  - Select most appropriate option based on requirements
+
+  **dataset_search:**
+  - Find datasets by query, tags, author
+  - Sort by downloads, likes, trending
+  - ALWAYS verify format with hub_repo_details before training
+  - Select most suitable dataset based on format and task
+
+  **paper_search:**
+  - Find research papers semantically
+  - Get paper abstracts and links
+  - Useful for understanding methods before implementing
+
+  **hub_repo_details:**
+  - Get detailed information about repos
+  - ⚠️ CRITICAL: Use this to verify dataset format before training
+  - Check model size, architecture, requirements
+  - Verify dataset columns, splits, size
+
+  **hf_whoami:**
+  - Check authentication status
+  - Verify token has correct permissions
+  - Use before operations requiring write access
+
+  ## Execution & Storage Tools
+
+  **hf_jobs:**
+  - Execute workloads on cloud infrastructure with detailed hardware specs (vCPU/RAM/GPU)
+  - ⚠️ Set timeout >30m (default too short)
+  - ⚠️ Include HF_TOKEN for Hub operations
+  - ⚠️ Storage is EPHEMERAL - must push_to_hub
+
+  **hf_private_repos:**
+  - Store job outputs persistently in datasets with push_to_hub (jobs lose files after completion)
+  - Upload logs, scripts, results that can't push_to_hub
+  - Create private repos for sensitive data
+  - Content-based: pass strings/bytes, not file paths
+  - After upload: provide repo URL to user
+
+  **plan_tool:**
+  - Break down complex tasks (3+ steps)
+  - Update frequently to show progress
+  - Exactly ONE task in_progress at a time
+  - Mark completed immediately after finishing
+
+  ## Space Tools (MCP)
+
+  **space_search:**
+  - Find deployed Spaces (demos, applications)
+  - Discover existing implementations
+
+  **use_space:**
+  - Give user access to a Space
+  - Returns link for user (may not be visible to you)
+
+  **dynamic_space:**
+  - Execute tasks using Space functionality
+  - Image generation, OCR, text-to-speech, etc.
+  - Only works with MCP-enabled Spaces
+
+  # Ground Rules for Reliability
+
+  ## Async Operations (Jobs, Long Tasks)
+
+  **✓ DO:**
+  - Poll logs automatically after submission to ensure job is running and works as expected
+  - Include Trackio dashboard URL for training jobs
+  - Note that user can check status later
+  - Explain what's happening in the background
+
+  **✗ DON'T:**
+  - Check status unless user asks
+  - Assume job will complete quickly
+
+  ## Resource Selection
+
+  **✓ DO:**
+  - Research and evaluate 3-5 options for models/datasets
+  - Assess key details (size, format, popularity, suitability)
+  - Select optimal option based on task requirements and efficiency
+  - ALWAYS validate dataset format matches training method before proceeding
+  - Choose hardware that balances cost and performance
+
+  **✗ DON'T:**
+  - Skip research and validation steps
+  - Assume most popular is automatically best for task
+  - Proceed with training without format validation
+  - Select unnecessarily expensive hardware without justification
+
+  ## Documentation Usage
+
+  **✓ DO:**
+  - Research before implementing any ML task
+  - Use explore → fetch → implement pattern
+  - Check current APIs and parameters
+  - Base implementation on researched approaches
+
+  **✗ DON'T:**
+  - Implement based on internal knowledge without checking docs
+  - Assume you know current API syntax
+  - Skip research for "simple" tasks
+  - Use outdated patterns or methods
+
+  ## Error Handling & Recovery
+
+  **When Errors Occur:**
+  1. ✅ Keep task in `in_progress` status (don't mark complete)
+  2. ✅ Create new todo for resolving the issue
+  3. ✅ Explain error clearly with technical details
+  4. ✅ Provide actionable solution based on error type
+  5. ✅ Check documentation if API/syntax error
+  6. ✅ Verify configuration if job fails
+  7. ✅ Implement fix and retry automatically with corrected approach
+
+  **Common Issues & Solutions:**
+
+  ### Job Timeout Exceeded
+  **Symptom:** Job stops mid-execution, incomplete
+  **Cause:** Timeout too short for workload
+  **Solution:**
+  ```python
+  # ✗ WRONG: Default timeout
+  {"timeout": "30m"}  # Too short for training!
+
+  # ✓ CORRECT: Appropriate timeout
+  {"timeout": "4h"}  # For 1-3B model training
+  {"timeout": "8h"}  # For 7-13B model training
+  ```
+
+  ### Model Not Pushed to Hub
+  **Symptom:** Training completes but model not on Hub
+  **Causes & Solutions:**
+  1. Missing `push_to_hub=True` in training config
+  2. Missing `hub_model_id` in training config
+  3. Missing `HF_TOKEN` in job env
+  4. Token lacks write permissions
+
+  **Solution:**
+  ```python
+  # Training config:
+  training_args = SFTConfig(
+      push_to_hub=True,  # ← Must be True
+      hub_model_id="username/model-name",  # ← Must be set
+      # ...
+  )
+
+  # Verify token: hf_whoami()
+  ```
+
+  ### Dataset Format Mismatch
+  **Symptom:** Training fails with KeyError or format errors
+  **Cause:** Dataset format doesn't match training method
+  **Solution:**
+  1. Use `hub_repo_details` to inspect dataset structure
+  2. Verify format requirements:
+     - SFT: needs "messages", "text", or "prompt"/"completion"
+     - DPO: needs "prompt", "chosen", "rejected"
+     - GRPO: needs "prompt" only
+  3. Preprocess dataset to correct format
+  4. Proceed with corrected configuration
+
+  ### Out of Memory (OOM)
+  **Symptom:** Job crashes with CUDA OOM error
+  **Solutions (in order of preference):**
+  1. Increase `gradient_accumulation_steps` (compensates smaller batch)
+  2. Reduce `per_device_train_batch_size` (try 4 → 2 → 1)
+  3. Enable `gradient_checkpointing=True`
+  4. Reduce `max_length` (e.g., 1024 → 512)
+  5. Upgrade to larger GPU (t4 → a10g → a100 → h100)
+
+  # Communication Style
+
+  - Be concise and direct
+  - Don't flatter the user
+  - Don't use emojis in regular communication (okay in status messages like "✅ Job submitted!")
+  - Don't use exclamation points in regular text
+  - If limited in a task, offer alternatives
+  - Don't thank user when they provide information
+  - Explain what you're doing for non-trivial operations
+  - Answer user questions directly - questions take precedence over task completion
+  - One-word answers when appropriate for simple questions
+  - For complex tasks, provide structured breakdown
+
+  # Examples
+
+  <example>
+  User: Fine-tune Llama for instruction following on ultrachat dataset
+
+  Assistant:
+  ✓ I'll help you fine-tune Llama for instruction following. Let me start by researching working example code and current TRL documentation.
+
+  [Creates plan with plan_tool: Find examples, Study code, Research docs, Find model, Validate dataset, Create script, Submit job]
+
+  [STEP 1: Find working example code FIRST]
+  github_find_examples({"repo": "trl", "keyword": "sft", "org": "huggingface"})
+  # Found: examples/scripts/sft.py, examples/scripts/sft_vlm.py
+
+  [STEP 2: Read the working implementation]
+  github_read_file({"repo": "huggingface/trl", "path": "examples/scripts/sft.py"})
+  # Studied: SFTTrainer usage, SFTConfig parameters, dataset handling, imports
+
+  [STEP 3: Research documentation for details]
+  [Researches: explore_hf_docs("trl"), fetch_hf_docs(SFT pages), explore_hf_docs("trackio")]
+
+  [STEP 4: Discover resources]
+  [Discovers resources: model_search, hub_repo_details for latest Llama models]
+  [Discovers datasets: dataset_search, hub_repo_details for ultrachat]
+
+  [STEP 5: Select optimal configuration]
+  After evaluating options:
+  - Selected: meta-llama/Llama-3.2-1B (1.24B params) - optimal balance of quality and efficiency
+  - Dataset: HuggingFaceH4/ultrachat_200k (207K samples, "messages" format ✓ SFT-compatible)
+  - Hardware: t4-small (4vCPU/15GB/GPU 16GB, $0.60/hr) - cost-efficient for this model size
+  - Estimated: 3 hours, ~$1.80 total cost
+
+  [STEP 6: Create and submit training job]
+  [Updates plan: mark resource selection complete, mark script creation in_progress]
+
+  [Creates script based on examples/scripts/sft.py pattern with:
+   - Imports from studied example (transformers, trl, datasets, trackio)
+   - SFTTrainer configuration from working code
+   - Dataset handling pattern from example (load_dataset + format verification)
+   - Trackio monitoring as shown in docs
+   - push_to_hub configuration with HF_TOKEN]
+
+  [Submits training job with hf_jobs: hardware=t4-small, timeout=4h, env=HF_TOKEN]
+
+  </example>
+
+  <example>
+  User: My Space crashes on startup
+
+  Assistant:
+  ✓ I'll help debug your Space. Let me inspect the repository and logs.
+
+  [Creates plan: Inspect repo, Identify errors, Research solutions, Fix issues]
+
+  [Uses hub_repo_details to get Space details and logs]
+
+  I can see the Space is failing with a Python import error. It's trying to import 'gradio' but the package isn't in requirements.txt.
+
+  [Explores documentation: explore_hf_docs("gradio"), search_hf_api_endpoints for spaces endpoints (e.g. seeing logs) ]
+
+  Based on the Gradio docs, I need to add gradio to requirements.txt and ensure the app.py file is correctly structured.
+
+  [Fixes using hf_private_repos: uploads corrected requirements.txt and app.py]
+
+  [Checks if the Space is running and works as expected]
+  </example>
+
+  <example>
+  User: Process the squad dataset - filter where context length > 100 chars and push to my Hub
+
+  Assistant:
+  ✓ I'll process the SQuAD dataset with your filter and push the results.
+
+  [Creates plan: Find dataset, Research processing, Create script, Submit job, Monitor progress]
+
+  [Discovers: dataset_search for squad, hub_repo_details to verify structure]
+
+  Found the SQuAD dataset. It has 'context' and 'question' columns.
+  I'll filter rows where len(context) > 100 characters.
+
+  [Researches: explore_hf_docs("datasets"), fetch_hf_docs for processing/filtering]
+
+  [Submits processing job with hf_jobs and makes sure to push the results to the Hub]
+
+  </example>
+
+  # Additional Instructions
+
+  - **Always use current information:** Find working examples with github_find_examples + check documentation before implementing; internal knowledge may be outdated
+  - **Example code first:** ALWAYS use github_find_examples + github_read_file before implementing ML tasks - real code shows current APIs and patterns
+  - **Search before building:** Use Hub search tools, GitHub code search, and documentation before creating custom solutions
+  - **Verify explicitly:** Never assume dataset schemas, column names, or API details; always check with hub_repo_details
+  - **Base on documented practices:** Implement using researched approaches from documentation, not general knowledge
+  - **Follow ML best practices:** Proper splits, reproducibility, evaluation metrics, suitable hardware
+  - **Respect storage boundaries:** Spaces and repos are permanent; job filesystems are ephemeral
+  - **Content-based operations:** For hf_private_repos, pass file contents not paths; local and remote filesystems are separate
+  - **Secure secrets:** HF_TOKEN automatically available via env; never expose or log tokens
+  - **Include links:** Provide direct URLs when referencing models, datasets, papers, jobs, repos
+  - **Execute user requests:** Always do what the user asks you to do
+  - **Parallel tool execution:** Call multiple independent tools simultaneously for efficiency when possible
+
+  # Token Count & Context Management
+
+  {{ num_tools }} tools are available. Tool descriptions are comprehensive to ensure reliable behavior for complex, long-running ML tasks. Prioritize:
+  1. Research current documentation before implementing
+  2. Validate resources before expensive operations
+  3. Handle async operations correctly
+  4. Ensure result persistence
+  5. Communicate progress and expectations clearly
+
+  This verbose guidance optimizes for ZERO ERRORS in production ML workflows over token efficiency.

agent/tools/docs_tools.py

@@ -509,10 +509,16 @@ async def hf_docs_fetch_handler(arguments: dict[str, Any]) -> tuple[str, bool]:
 EXPLORE_HF_DOCS_TOOL_SPEC = {
     "name": "explore_hf_docs",
     "description": (
-        "Explore the Hugging Face documentation at a glance. "
-        "Select an endpoint from the available options and get a list of all documentation pages "
-        "with their titles, URLs, and a 300-character glimpse of each page. "
-        "Use this to discover what documentation is available before fetching specific pages."
+        "Explore Hugging Face documentation structure and discover available pages with 300-character previews. "
+        "⚠️ MANDATORY: ALWAYS use this BEFORE implementing any ML task (training, fine-tuning, data processing, inference). "
+        "Your training data may be outdated - current documentation is the source of truth. "
+        "**Use when:** (1) Starting any implementation task, (2) User asks 'how to' questions, "
+        "(3) Before writing training/processing code, (4) Researching library capabilities, "
+        "(5) Verifying API syntax and parameters. "
+        "**Pattern:** explore (discover structure) → fetch_hf_docs (get details) → implement with researched approach. "
+        "Returns: Sidebar navigation with titles, URLs, and glimpses of all pages in the selected documentation. "
+        "**Then:** Use fetch_hf_docs with specific URLs from results to get full content. "
+        "**Critical for reliability:** Never implement based on internal knowledge without checking current docs first - APIs change frequently."
     ),
     "parameters": {
         "type": "object",
@@ -645,10 +651,16 @@ EXPLORE_HF_DOCS_TOOL_SPEC = {
 HF_DOCS_FETCH_TOOL_SPEC = {
     "name": "fetch_hf_docs",
     "description": (
-        "Fetch the full content of a specific HF documentation page. "
-        "Provide the full URL to the doc page (e.g., from explore_hf_docs results). "
-        "Returns the complete markdown content of that page. "
-        "Use explore_hf_docs first to discover available pages."
+        "Fetch full markdown content of a specific HF documentation page. "
+        "⚠️ CRITICAL: Use this after explore_hf_docs to get detailed implementation guidance. "
+        "**Use when:** (1) Found relevant page in explore_hf_docs results, (2) Need complete API documentation, "
+        "(3) Need training method details (SFT/DPO/GRPO), (4) Need configuration examples, "
+        "(5) Need parameter descriptions and usage patterns. "
+        "**Pattern:** explore_hf_docs (find relevant page) → fetch_hf_docs (get full content) → implement using documented approach. "
+        "Provide full URL from explore_hf_docs results (e.g., 'https://huggingface.co/docs/trl/sft_trainer'). "
+        "Returns: Complete markdown documentation with examples, parameters, and usage patterns. "
+        "**For training tasks:** ALWAYS fetch trainer docs (SFTConfig, DPOConfig, etc.) before creating training scripts. "
+        "**Critical for reliability:** This ensures you use current APIs and best practices."
     ),
     "parameters": {
         "type": "object",
@@ -678,9 +690,15 @@ async def _get_api_search_tool_spec() -> dict[str, Any]:
     return {
         "name": "search_hf_api_endpoints",
         "description": (
-            "Search the HuggingFace OpenAPI specification by tag to find related API endpoints. "
-            "Returns all endpoints with the specified tag including curl examples showing how to use them. "
-            "Each result includes the endpoint path, summary, usage example with curl, and response information."
+            "Search HuggingFace OpenAPI specification by tag to find API endpoints with curl examples. "
+            "**Use when:** (1) Need to interact with HF Hub API directly, (2) Building scripts for repo operations, "
+            "(3) Need authentication patterns, (4) Understanding API parameters and responses, "
+            "(5) Need curl examples for HTTP requests. "
+            "Returns: Endpoint paths, methods, parameters, curl examples with authentication, and response schemas. "
+            "**Pattern:** search_hf_api_endpoints (find endpoint) → use curl pattern in implementation. "
+            "Tags group related operations: repos, models, datasets, inference, spaces, etc. "
+            "**Note:** Each result includes curl example with $HF_TOKEN placeholder for authentication. "
+            "**For tool building:** This provides the API foundation for creating Hub interaction scripts."
         ),
         "parameters": {
             "type": "object",

agent/tools/github_find_examples.py

@@ -404,47 +404,57 @@ def find_examples(
 # Tool specification
 GITHUB_FIND_EXAMPLES_TOOL_SPEC = {
     "name": "github_find_examples",
-    "description": "Discover best practices, reusable scripts, tutorials, and demos for using a specific library or framework. This is an important step before implementing anything ML related. "
-    "Use together with github_read_file tool.\n\n"
-    "## When to use this tool\n\n"
-    "- ALWAYS before implementing any training/inference/benchmarking or other ML related code or answering how-to question.\n"
-    "- When exploring a new repository and need to understand how to use it\n"
-    "## How it works\n\n"
-    "1. Fetches all (examples, tutorials, demos, notebooks, scripts, etc.) from the repository\n"
-    "2. If keyword provided, scores found files against the keyword using fuzzy matching\n"
-    "3. Returns best matches sorted by relevance score\n"
-    "## Examples\n\n"
-    "<example>\n"
-    "// ML Workflow Step: Find GRPO/SFT/DPO/RLOO etc training examples\n"
-    "// Task: Starting GRPO fine-tuning project, need reference implementations\n"
-    "{\n"
-    "  keyword: 'grpo',\n"
-    "  repo: 'trl',\n"
-    "  org: 'huggingface'\n"
-    "}\n"
-    "// Returns: examples/scripts/grpo_agent.py, examples/scripts/grpo_vlm.py\n"
-    "// Next step: Use github_read_file to study the implementation\n"
-    "</example>\n\n"
-    "<example>\n"
-    "// ML Workflow Step: Discover all training examples in TRL\n"
-    "// Task: Exploring available training methods before choosing approach\n"
-    "{\n"
-    "  repo: 'trl',\n"
-    "  org: 'huggingface',\n"
-    "  max_results: 20\n"
-    "}\n"
-    "// Lists all example scripts: PPO, DPO, GRPO, reward modeling, etc.\n"
-    "</example>\n\n"
-    "<example>\n"
-    "// ML Workflow Step: Find LoRA fine-tuning examples\n"
-    "// Task: Learning parameter-efficient fine-tuning with PEFT\n"
-    "{\n"
-    "  keyword: 'lora',\n"
-    "  repo: 'peft',\n"
-    "  org: 'huggingface'\n"
-    "}\n"
-    "// Discovers LoRA configuration and training examples\n"
-    "</example>",
+    "description": (
+        "Discover working code examples, tutorials, scripts, and demos in GitHub repositories. "
+        "⚠️ CRITICAL: ALWAYS use this BEFORE implementing ML tasks - find working reference code first. "
+        "Your training data may be outdated; real repository examples show current best practices. "
+        "**Use when:** (1) Starting any ML implementation (training, inference, evaluation), "
+        "(2) User asks 'how to' questions about libraries, (3) Need reference implementations, "
+        "(4) Exploring library capabilities, (5) Before writing training/processing scripts. "
+        "**Pattern:** github_find_examples (discover) → github_read_file (study code) → implement with researched approach. "
+        "Returns: List of example files (scripts/notebooks/tutorials) with paths and URLs, sorted by relevance. "
+        "**Then:** Use github_read_file to read the actual implementation code. "
+        "**Critical for reliability:** Real examples prevent outdated API usage and show proven patterns. "
+        "## How it works\n\n"
+        "1. Fetches all example files (examples/, scripts/, tutorials/, demos/, notebooks/, etc.) from repository\n"
+        "2. If keyword provided, scores files against keyword using fuzzy matching\n"
+        "3. Returns best matches sorted by relevance and pattern priority\n"
+        "4. Provides copyable parameters for github_read_file tool\n\n"
+        "## Examples\n\n"
+        "<example>\n"
+        "// ML Workflow Step: Find GRPO training examples before implementation\n"
+        "// Task: Starting GRPO fine-tuning project, need reference implementation\n"
+        "{\n"
+        "  keyword: 'grpo',\n"
+        "  repo: 'trl',\n"
+        "  org: 'huggingface'\n"
+        "}\n"
+        "// Returns: examples/scripts/grpo_agent.py, examples/scripts/grpo_vlm.py\n"
+        "// Next step: github_read_file to study working implementation\n"
+        "</example>\n\n"
+        "<example>\n"
+        "// ML Workflow Step: Discover all available training methods\n"
+        "// Task: Exploring TRL training options before choosing approach\n"
+        "{\n"
+        "  repo: 'trl',\n"
+        "  org: 'huggingface',\n"
+        "  max_results: 20\n"
+        "}\n"
+        "// Lists: SFT, DPO, GRPO, PPO, reward modeling examples\n"
+        "// Helps user choose appropriate method\n"
+        "</example>\n\n"
+        "<example>\n"
+        "// ML Workflow Step: Find LoRA fine-tuning examples\n"
+        "// Task: Learning parameter-efficient fine-tuning patterns\n"
+        "{\n"
+        "  keyword: 'lora',\n"
+        "  repo: 'peft',\n"
+        "  org: 'huggingface'\n"
+        "}\n"
+        "// Discovers LoRA configuration and training examples\n"
+        "// Shows current PEFT API usage patterns\n"
+        "</example>"
+    ),
     "parameters": {
         "type": "object",
         "properties": {

agent/tools/github_list_repos.py

@@ -202,13 +202,19 @@ def list_repos(
 GITHUB_LIST_REPOS_TOOL_SPEC = {
     "name": "github_list_repos",
     "description": (
-        "List and discover repositories for any GitHub user or organization with flexible sorting.\n\n"
-        "Returns comprehensive repository information including stars, forks, language, topics, and direct URLs. "
-        "Sorts by stars, forks, update date, or creation date.\n\n"
+        "List and discover repositories for GitHub organizations or users with flexible sorting. "
+        "**Use when:** (1) Exploring what libraries exist for a task, (2) Finding the right library to use, "
+        "(3) Discovering popular or active projects, (4) Checking recently updated repos for latest features, "
+        "(5) Finding alternative libraries in an organization. "
+        "**Pattern:** github_list_repos (discover libraries) → github_find_examples (find usage examples) → implement. "
+        "Returns: Comprehensive repository information (stars, forks, language, topics, URLs), sorted by preference. "
+        "**Then:** Use github_find_examples on selected repo to discover example code. "
+        "Sorts by: stars (popularity), forks (community), updated (activity), created (age).\n\n"
         "## When to use this tool\n\n"
-        "- When you need to find libraries to use in your implementation, or to explore what repositories exist for a task.\n"
-        "- When debugging an error to looking up if others are having the same issues in repositories."
-        "- When finding the most popular or active projects for a user or org\n"
+        "- When you need to find libraries to use in your implementation\n"
+        "- When exploring what repositories exist for a task or domain\n"
+        "- When debugging an error and looking up if others have similar issues in repos\n"
+        "- When finding the most popular or actively maintained projects for a user/org\n"
         "## Examples\n\n"
         "<example>\n"
         "// ML Workflow Step: Discover HF libraries for RLHF/alignment\n"

agent/tools/github_read_file.py

@@ -250,39 +250,50 @@ def read_file(
 GITHUB_READ_FILE_TOOL_SPEC = {
     "name": "github_read_file",
     "description": (
-        "Read file contents from any GitHub repository with line range support.\n\n"
-        "Fetches exact file contents in the given line range (default 300 lines, use line_start/line_end adjust). \n\n"
+        "Read file contents from GitHub repositories with line range support (default 300 lines). "
+        "⚠️ CRITICAL: Use AFTER github_find_examples to study working implementation code. "
+        "**Use when:** (1) Found example file via github_find_examples and need full code, "
+        "(2) Need to read trainer class implementation, (3) Study configuration patterns, "
+        "(4) Read specific code sections with line ranges, (5) Review code from specific branches/commits. "
+        "**Pattern:** github_find_examples (discover files) → github_read_file (read code) → implement using researched patterns. "
+        "Returns: File contents with line numbers, formatted for LLM reading. Auto-converts Jupyter notebooks to markdown. "
+        "**Then:** Implement using patterns and APIs from the example code. "
+        "**Critical for reliability:** Reading working examples prevents API errors and shows current best practices. "
+        "Use line_start/line_end for large files (>300 lines) to read specific sections.\n\n"
         "## When to use this tool\n\n"
-        "- When reading example code, implementations, or documentation on a specific github file\n"
-        "- When you found a file via github_list_repos, or github_find_examples and need its contents\n"
+        "- When reading example code, trainer implementations, or configuration files\n"
+        "- After github_find_examples returns file paths you want to study\n"
         "- When investigating specific code sections with line ranges\n"
-        "- When reading from specific branches, tags, or commits\n"
+        "- When reading from specific branches, tags, or commits (use ref parameter)\n\n"
         "## When NOT to use this tool\n\n"
-        "- When you don't know the exact file path beforehand (use github_search_code or github_find_examples first)\n\n"
+        "- When you don't know exact file path (use github_find_examples or github_search_code first)\n"
+        "- When searching for code patterns across repos (use github_search_code instead)\n\n"
         "## Examples\n\n"
         "<example>\n"
-        "// ML Workflow Step: Reading example code from for GRPO training with TRL\n"
-        "// Use case: Read trainer class to understand API and methods\n"
+        "// ML Workflow Step: Read GRPO trainer class after finding via github_find_examples\n"
+        "// Use case: Understand GRPOTrainer API, parameters, and methods\n"
         "{\n"
         "  repo: 'huggingface/trl',\n"
         "  path: 'trl/trainer/grpo_trainer.py',\n"
         "  line_start: 1,\n"
         "  line_end: 200\n"
         "}\n"
-        "// Read class definition and constructor to understand parameters\n"
+        "// Read class definition and constructor to understand current API\n"
+        "// Shows: __init__ parameters, configuration, required arguments\n"
         "</example>\n\n"
         "<example>\n"
-        "// ML Workflow Step: Study complete training script\n"
-        "// Use case: Learn end-to-end VLM fine-tuning with GRPO\n"
+        "// ML Workflow Step: Study complete training script from examples\n"
+        "// Use case: Learn end-to-end VLM fine-tuning workflow\n"
         "{\n"
         "  repo: 'huggingface/trl',\n"
         "  path: 'examples/scripts/grpo_vlm.py'\n"
         "}\n"
-        "// Returns first 300 lines of the file\n"
+        "// Returns first 300 lines - shows full training setup\n"
+        "// Use line_start/line_end if need to read more\n"
         "</example>\n\n"
         "<example>\n"
-        "// ML Workflow Step: Check configuration patterns\n"
-        "// Use case: Learn how to structure training configs\n"
+        "// ML Workflow Step: Check TrainingArguments configuration patterns\n"
+        "// Use case: Learn how to structure training configs correctly\n"
         "{\n"
         "  repo: 'huggingface/transformers',\n"
         "  path: 'examples/pytorch/language-modeling/run_clm.py',\n"
@@ -290,6 +301,7 @@ GITHUB_READ_FILE_TOOL_SPEC = {
         "  line_end: 150\n"
         "}\n"
         "// Read argument parsing and config setup section\n"
+        "// Shows: current parameter names, default values, best practices\n"
         "</example>"
     ),
     "parameters": {

agent/tools/github_search_code.py

@@ -334,19 +334,25 @@ def search_code(
 GITHUB_SEARCH_CODE_TOOL_SPEC = {
     "name": "github_search_code",
     "description": (
-        "Search for code patterns across GitHub repositories with intelligent pattern matching.\n\n"
-        "Searches for specific code patterns, functions, classes, or implementations across GitHub. "
-        "Intelligently maps patterns to GitHub's Code Search API for efficient server-side filtering, "
-        "with automatic client-side filtering for complex patterns. Returns code snippets with context.\n\n"
+        "Search for specific code patterns, functions, or classes across GitHub repositories. "
+        "**Use when:** (1) Need to find specific function/class implementations, "
+        "(2) Looking for how specific APIs are used across repos, (3) Searching for specific patterns or methods, "
+        "(4) Investigating feature implementations across different projects, (5) Finding usage examples of specific imports or calls. "
+        "**Pattern:** github_search_code (find usage) → github_read_file (read full context) → understand implementation. "
+        "Returns: Code snippets with line numbers, file paths, and repo URLs. Intelligently maps patterns to GitHub API. "
+        "**Then:** Use github_read_file to read full file context. "
+        "**vs github_find_examples:** Use search_code for specific code patterns (e.g., 'AutoModelForCausalLM.from_pretrained'); "
+        "use find_examples for discovering tutorial/example files. "
+        "Supports regex searches for advanced patterns.\n\n"
         "## When to use this tool\n\n"
         "- When searching for specific code patterns, functions, or classes across repositories\n"
         "- When looking for implementation examples of specific methods or APIs\n"
         "- When you need to find where specific code exists across multiple files or repos\n"
         "- When investigating how a feature is implemented in different repositories\n"
         "- When searching for TODO comments, specific patterns, or code structures\n"
-        "- Use this for searching actual implementation code (not examples - use github_find_examples for those)\n\n"
+        "- Use this for searching actual implementation code (not example files - use github_find_examples for those)\n\n"
         "## When NOT to use this tool\n\n"
-        "- When looking for example files or tutorials (use github_find_examples instead)\n"
+        "- When looking for example/tutorial files (use github_find_examples instead)\n"
         "- When you already know the exact file path (use github_read_file directly)\n"
         "- When you need to list repositories (use github_list_repos instead)\n\n"
         "## Repository Patterns\n\n"

agent/tools/jobs_tool.py

@@ -790,31 +790,54 @@ To inspect, call this tool with `{{"operation": "scheduled inspect", "scheduled_
 HF_JOBS_TOOL_SPEC = {
     "name": "hf_jobs",
     "description": (
-        "Run Python scripts or Docker containers on HF cloud GPUs/CPUs.\n\n"
-        "## Operations:\n"
-        "run, ps, logs, inspect, cancel, scheduled run, scheduled ps, scheduled inspect, scheduled delete, scheduled suspend, scheduled resume\n\n"
-        "## Two modes:\n"
-        "1. **Python mode:** Provide 'script' + 'dependencies' → auto-handles pip install\n"
-        "2. **Docker mode:** Provide 'image' + 'command' → full control\n"
+        "Execute Python scripts or Docker containers on HF cloud infrastructure (CPUs/GPUs). "
+        "⚠️ CRITICAL for reliability: (1) Jobs run ASYNC - provide monitoring URL immediately, don't poll; "
+        "(2) Set timeout >30min (default too short - training needs 2-8h); "
+        "(3) HF_TOKEN auto-loaded to secrets for Hub ops (push_to_hub, private repos);"
+        "(4) Job storage EPHEMERAL - MUST push_to_hub() or ALL work is LOST. "
+        "**Use when:** User wants cloud compute, training models, data processing, batch inference, GPU workloads, scheduled tasks. "
+        "ALWAYS use this tool (✓), never bash 'hf jobs' commands (✗). Pass script content inline (✓), don't save to files unless requested (✗). "
+        "\n\n"
+        "**Operations:** run, ps, logs, inspect, cancel, scheduled run, scheduled ps, scheduled inspect, scheduled delete, scheduled suspend, scheduled resume. "
+        "\n\n"
+        "**Two Modes:**\n"
+        "1. Python mode: 'script' + 'dependencies' (UV with PEP 723 recommended for inline deps)\n"
+        "2. Docker mode: 'image' + 'command' (full environment control)\n"
         "(script and command are mutually exclusive)\n\n"
-        "## Available Hardware (vCPU/RAM/GPU):\n"
-        f"CPU: {CPU_FLAVORS_DESC}\n"
-        f"GPU: {GPU_FLAVORS_DESC}\n"
-        "## Examples:\n\n"
-        "**Fine-tune LLM and push to Hub:**\n"
-        "{'operation': 'run', 'script': 'from transformers import AutoModelForCausalLM, AutoTokenizer, Trainer\\nmodel = AutoModelForCausalLM.from_pretrained(\"Qwen/Qwen3-4B-Thinking-2507\")\\n# ... training code ...\\nmodel.push_to_hub(\"user-name/my-finetuned-model\")', 'dependencies': ['transformers', 'torch', 'datasets'], 'hardware_flavor': 'a10g-large', 'timeout': '4h', 'env': {'CUSTOM_VAR': 'value'}}\n\n"
-        "**Generate dataset daily and upload:**\n"
-        "{'operation': 'scheduled run', 'script': 'from datasets import Dataset\\nimport pandas as pd\\n# scrape/generate data\\ndf = pd.DataFrame(data)\\nds = Dataset.from_pandas(df)\\nds.push_to_hub(\"user-name/daily-dataset\")', 'dependencies': ['datasets', 'pandas'], 'schedule': '@daily'}\n\n"
-        "**Run custom training with Docker:**\n"
+        "**Available Hardware (vCPU/RAM/GPU):**\n"
+        f"• CPU: {CPU_FLAVORS_DESC}\n"
+        f"• GPU: {GPU_FLAVORS_DESC}\n"
+        "  ◦ Common: t4-small ($0.60/hr, demos/1-3B models), a10g-small ($1/hr), a10g-large ($2/hr, production 7-13B), a100-large ($4/hr, 30B+), h100 ($6/hr, 70B+)\n\n"
+        "**After Submission Ground Rules:**\n"
+        "✓ Return immediately with job ID and monitoring URL\n"
+        "✓ Provide expected completion time and cost estimate\n"
+        "✓ For training: Include Trackio dashboard URL\n"
+        "✓ Note user can check status later\n"
+        "✗ DON'T poll logs automatically\n"
+        "✗ DON'T wait for completion\n"
+        "✗ DON'T check status unless user asks\n\n"
+        "**For Training Tasks:**\n"
+        "• ALWAYS research TRL docs first: explore_hf_docs('trl') → fetch_hf_docs(<trainer_url>)\n"
+        "• ALWAYS validate dataset format with hub_repo_details (SFT needs messages/text, DPO needs chosen/rejected)\n"
+        "• ALWAYS include Trackio monitoring in script (explore_hf_docs('trackio'))\n"
+        "• ALWAYS enable push_to_hub=True in training config\n"
+        "• Set timeout 2-8h for training (NOT default 30m)\n"
+        "• Confirm model/dataset choices with user before submitting\n\n"
+        "**Examples:**\n\n"
+        "**Training - Fine-tune LLM:**\n"
+        "{'operation': 'run', 'script': '# Training script with TRL\\nfrom trl import SFTConfig, SFTTrainer\\nfrom transformers import AutoModelForCausalLM\\nmodel = AutoModelForCausalLM.from_pretrained(\"Qwen/Qwen3-4B\")\\n# ... researched implementation from docs ...\\ntrainer.train()\\ntrainer.push_to_hub(\"user-name/my-model\")', 'dependencies': ['transformers', 'trl', 'torch', 'datasets', 'trackio'], 'hardware_flavor': 'a10g-large', 'timeout': '4h'}\n\n"
+        "**Data Processing:**\n"
+        "{'operation': 'run', 'script': 'from datasets import load_dataset\\nds = load_dataset(\"data\")\\n# process...\\nds.push_to_hub(\"user/processed\")', 'dependencies': ['datasets', 'pandas'], 'hardware_flavor': 'cpu-upgrade', 'timeout': '2h'}\n\n"
+        "**Scheduled Daily Job:**\n"
+        "{'operation': 'scheduled run', 'schedule': '@daily', 'script': 'from datasets import Dataset\\nimport pandas as pd\\n# scrape/generate data\\ndf = pd.DataFrame(data)\\nds = Dataset.from_pandas(df)\\nds.push_to_hub(\"user-name/daily-dataset\")', 'dependencies': ['datasets', 'pandas'], 'hardware_flavor': 'cpu-basic'}\n\n"
+        "**Docker Mode:**\n"
         "{'operation': 'run', 'image': 'pytorch/pytorch:2.0.0-cuda11.7-cudnn8-runtime', 'command': ['python', 'train.py', '--epochs', '10'], 'hardware_flavor': 'a100-large'}\n\n"
-        "**Monitor jobs:**\n"
-        "{'operation': 'ps'} - list running\n"
-        "{'operation': 'logs', 'job_id': 'xxx'} - stream logs\n"
-        "{'operation': 'cancel', 'job_id': 'xxx'} - stop job\n\n"
-        "## CRITICAL: Files are ephemeral!\n"
-        "Everything created during execution is DELETED when job finishes. Always .push_to_hub() your outputs (models, datasets, artifacts) in the script.\n\n"
-        "## After job completion:\n"
-        "If needed or asked by the user, use hf_private_repos tool to store scripts/logs/results to Hub for persistent storage."
+        "**Monitor Operations:**\n"
+        "{'operation': 'ps'} - List all jobs\n"
+        "{'operation': 'logs', 'job_id': 'xxx'} - Stream logs (only when user requests)\n"
+        "{'operation': 'inspect', 'job_id': 'xxx'} - Get job details\n"
+        "{'operation': 'cancel', 'job_id': 'xxx'} - Stop job\n\n"
+        "⚠️ CRITICAL: Files created during execution are DELETED when job finishes. MUST push_to_hub() all outputs (models, datasets, artifacts) in script. For logs/scripts, use hf_private_repos after completion."
     ),
     "parameters": {
         "type": "object",

agent/tools/plan_tool.py

@@ -74,7 +74,20 @@ def get_current_plan() -> List[Dict[str, str]]:
 # Tool specification
 PLAN_TOOL_SPEC = {
     "name": "plan_tool",
-    "description": "Manage a plan with a list of todos. Each call replaces the entire plan with the provided todos list.",
+    "description": (
+        "Manage task planning and progress tracking with todo list (pending/in_progress/completed statuses). "
+        "⚠️ CRITICAL: ALWAYS use for multi-step tasks (3+ steps) and MUST update frequently to show progress. "
+        "**Use when:** (1) User provides multiple tasks, (2) Complex workflows (training, evaluation, data processing), "
+        "(3) Tasks requiring multiple tool calls, (4) Need to communicate progress clearly to user, "
+        "(5) Breaking down ambiguous requests into concrete steps. "
+        "**Pattern:** Create plan at start → Mark in_progress when starting task → Mark completed immediately after finishing → User sees clear progress. "
+        "Each call replaces entire plan (full list required). "
+        "**Critical for reliability:** Exactly ONE task in_progress at a time (not zero, not multiple). "
+        "Mark tasks completed IMMEDIATELY after finishing - don't batch completions. "
+        "**For long-running tasks:** Update plan after each major step to keep user informed. "
+        "**Only mark completed when:** Task fully accomplished, no errors, all requirements met. "
+        "Keep tasks pending if blocked/errors occur - create new task to resolve blockers."
+    ),
     "parameters": {
         "type": "object",
         "properties": {

agent/tools/private_hf_repo_tools.py

@@ -16,7 +16,9 @@ from huggingface_hub.utils import HfHubHTTPError
 from agent.tools.types import ToolResult
 
 # Operation names
-OperationType = Literal["upload_file", "create_repo", "check_repo", "list_files", "read_file"]
+OperationType = Literal[
+    "upload_file", "create_repo", "check_repo", "list_files", "read_file"
+]
 
 
 async def _async_call(func, *args, **kwargs):
@@ -33,7 +35,7 @@ def _build_repo_url(repo_id: str, repo_type: str = "dataset") -> str:
 def _content_to_bytes(content: str | bytes) -> bytes:
     """Convert string or bytes content to bytes."""
     if isinstance(content, str):
-        return content.encode('utf-8')
+        return content.encode("utf-8")
     return content
 
 
@@ -159,7 +161,20 @@ Call this tool with:
   }
 }
 ```
-Note: Repositories are always created as private.
+
+### Create a Space
+Call this tool with:
+```json
+{
+  "operation": "create_repo",
+  "args": {
+    "repo_id": "my-gradio-app",
+    "repo_type": "space",
+    "space_sdk": "gradio"
+  }
+}
+```
+Note: Repositories are always created as private. For spaces, `space_sdk` is required (gradio, streamlit, static, or docker).
 
 ### Check if a repository exists
 Call this tool with:
@@ -261,13 +276,15 @@ Call this tool with:
 
             # Create repo if needed
             if not repo_exists and create_if_missing:
-                await self._create_repo(
-                    {
-                        "repo_id": repo_id,
-                        "repo_type": repo_type,
-                        "private": True,
-                    }
-                )
+                create_args = {
+                    "repo_id": repo_id,
+                    "repo_type": repo_type,
+                    "private": True,
+                }
+                # Pass through space_sdk if provided (required for spaces)
+                if "space_sdk" in args:
+                    create_args["space_sdk"] = args["space_sdk"]
+                await self._create_repo(create_args)
             elif not repo_exists:
                 return {
                     "formatted": f"Repository {repo_id} does not exist. Set create_if_missing: true to create it.",
@@ -332,6 +349,7 @@ Call this tool with:
 
         repo_type = args.get("repo_type", "dataset")
         private = True  # Always create private repos
+        space_sdk = args.get("space_sdk")  # Required if repo_type is "space"
 
         try:
             # Check if repo already exists
@@ -347,14 +365,27 @@ Call this tool with:
                     "resultsShared": 1,
                 }
 
+            # Validate space_sdk for spaces
+            if repo_type == "space" and not space_sdk:
+                return {
+                    "formatted": "space_sdk is required when creating a space. Valid values: gradio, streamlit, static, docker",
+                    "totalResults": 0,
+                    "resultsShared": 0,
+                    "isError": True,
+                }
+
             # Create repository
-            repo_url = await _async_call(
-                self.api.create_repo,
-                repo_id=repo_id,
-                repo_type=repo_type,
-                private=private,
-                exist_ok=True,
-            )
+            create_kwargs = {
+                "repo_id": repo_id,
+                "repo_type": repo_type,
+                "private": private,
+                "exist_ok": True,
+            }
+            # Add space_sdk only for spaces
+            if repo_type == "space" and space_sdk:
+                create_kwargs["space_sdk"] = space_sdk
+
+            repo_url = await _async_call(self.api.create_repo, **create_kwargs)
 
             response = f"""✓ Repository created successfully!
 
@@ -565,18 +596,30 @@ To create it, call this tool with:
 PRIVATE_HF_REPO_TOOL_SPEC = {
     "name": "hf_private_repos",
     "description": (
-        "Manage private Hugging Face repositories. "
-        "PRIMARY USE: Store job outputs, scripts, and logs from HF Jobs (ephemeral results need persistent storage). "
-        "SECONDARY USE: Read back stored files and list repo contents. "
-        "Pass file content as strings/bytes (no filesystem needed). "
-        "Call with no operation for full usage instructions."
+        "Manage private HF repositories - create, upload, read, list files in models/datasets/spaces. "
+        "⚠️ PRIMARY USE: Store job outputs persistently (job storage is EPHEMERAL - everything deleted after completion). "
+        "**Use when:** (1) Job completes and need to store logs/scripts/results, (2) Creating repos for training outputs, "
+        "(3) Reading back stored files, (4) Managing Space files, (5) Organizing job artifacts by path. "
+        "**Pattern:** hf_jobs (ephemeral) → hf_private_repos upload_file (persistent) → can read_file later. "
+        "ALWAYS pass file_content as string/bytes (✓), never file paths (✗) - this is content-based, no filesystem access. "
+        "**Operations:** create_repo (new private repo), upload_file (store content), read_file (retrieve content), list_files (browse), check_repo (verify exists). "
+        "**Critical for reliability:** Jobs lose all files after completion - use this tool to preserve important outputs. "
+        "Repositories created are ALWAYS private by default (good for sensitive training data/models). "
+        "For Spaces: must provide space_sdk ('gradio', 'streamlit', 'static', 'docker') when creating. "
+        "**Then:** After uploading, provide user with repository URL for viewing/sharing."
     ),
     "parameters": {
         "type": "object",
         "properties": {
             "operation": {
                 "type": "string",
-                "enum": ["upload_file", "create_repo", "check_repo", "list_files", "read_file"],
+                "enum": [
+                    "upload_file",
+                    "create_repo",
+                    "check_repo",
+                    "list_files",
+                    "read_file",
+                ],
                 "description": (
                     "Operation to execute. Valid values: [upload_file, create_repo, check_repo, list_files, read_file]"
                 ),
@@ -586,7 +629,8 @@ PRIVATE_HF_REPO_TOOL_SPEC = {
                 "description": (
                     "Operation-specific arguments as a JSON object. "
                     "Write ops: file_content (string/bytes), path_in_repo (string), repo_id (string), "
-                    "repo_type (dataset/model/space), create_if_missing (boolean), commit_message (string). "
+                    "repo_type (dataset/model/space), create_if_missing (boolean), commit_message (string), "
+                    "space_sdk (gradio/streamlit/static/docker - required when repo_type=space). "
                     "Read ops: repo_id (string), path_in_repo (for read_file), repo_type (optional)."
                 ),
                 "additionalProperties": True,

agent/tools/utils_tools.py

@@ -163,10 +163,13 @@ Common timezones: Europe/Paris, America/New_York, America/Los_Angeles, Asia/Toky
 UTILS_TOOL_SPEC = {
     "name": "utils",
     "description": (
-        "Utility operations for system information. "
-        "Get current date (dd-mm-yyyy) and time (HH:MM:SS.mmm) with timezone support. "
-        "Default timezone: Paris (Europe/Paris). "
-        "Call with no operation for full usage instructions."
+        "System utility operations - currently provides date/time with timezone support. "
+        "**Use when:** (1) Need current date for logging/timestamps, (2) User asks 'what time is it', "
+        "(3) Need timezone-aware datetime for scheduling/coordination, (4) Creating timestamped filenames. "
+        "**Operation:** get_datetime with optional timezone parameter (default: Europe/Paris). "
+        "Returns: Date (dd-mm-yyyy), time (HH:MM:SS.mmm), timezone info, ISO format, Unix timestamp. "
+        "**Pattern:** utils get_datetime → use timestamp in filename/log → upload to hf_private_repos. "
+        "Supports IANA timezone names: 'Europe/Paris', 'America/New_York', 'Asia/Tokyo', 'UTC'."
     ),
     "parameters": {
         "type": "object",

agent/utils/reliability_checks.py

@@ -0,0 +1,16 @@
+"""Reliability checks for job submissions and other operations"""
+
+from agent.utils.terminal_display import Colors
+
+
+def check_training_script_save_pattern(script: str) -> str | None:
+    """Check if a training script properly saves models."""
+    has_from_pretrained = "from_pretrained" in script
+    has_push_to_hub = "push_to_hub" in script
+
+    if has_from_pretrained and not has_push_to_hub:
+        return f"\n{Colors.RED}WARNING: We've detected that no model will be saved at the end of this training script. Please ensure this is what you want.{Colors.RESET}"
+    elif has_from_pretrained and has_push_to_hub:
+        return f"\n{Colors.GREEN}We've detected that a model will be pushed to hub at the end of this training.{Colors.RESET}"
+
+    return None