"""
Publish ECHO ULTIMATE as a HuggingFace Space (Docker SDK, Python 3.11).
Usage:
python scripts/publish_space.py --token YOUR_HF_TOKEN
python scripts/publish_space.py --token YOUR_HF_TOKEN --repo your-username/echo-ultimate
"""
import argparse
import os
import shutil
import sys
import tempfile
from pathlib import Path
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
_SPACE_README = """\
---
title: ECHO ULTIMATE
emoji: 🧠
colorFrom: blue
colorTo: purple
sdk: docker
pinned: true
license: apache-2.0
---
# ECHO ULTIMATE
### Metacognitive Calibration RL Environment
**The first open-source RL environment for training LLMs to know what they don't know.**
ECHO ULTIMATE teaches language models to accurately predict their own confidence —
solving the overconfidence problem that makes LLMs unreliable in high-stakes settings.
## What's Inside
| Tab | Feature |
|-----|---------|
| 🎯 Live Challenge | Answer questions with a confidence slider — see your calibration score in real time |
| 🤖 ECHO vs AI | Side-by-side comparison: calibrated ECHO vs overconfident baseline |
| 🧬 Epistemic Fingerprint | Radar chart of per-domain calibration accuracy |
| 📊 Training Evidence | All 6 plots from GRPO training — ECE curves, reward curves, reliability diagrams |
| 🏆 Official Evaluation | Run the 3 OpenEnv benchmark tasks |
| ⚡ Live Training | Watch ECE drop in real-time as GRPO trains |
## How It Works
ECHO uses **GRPO (Group Relative Policy Optimization)** with a custom reward function:
```
R = accuracy_reward − overconfidence_penalty
```
The agent learns to output `75Paris` —
pairing every answer with a calibrated probability estimate.
## EchoBench Dataset
The 7-domain benchmark: [Vikaspandey582003/echobench](https://huggingface.co/datasets/Vikaspandey582003/echobench)
| Domain | Source |
|--------|--------|
| Math | GSM8K |
| Logic | AI2-ARC |
| Factual | TriviaQA |
| Science | SciQ |
| Medical | MedMCQA |
| Coding | Synthetic |
| Creative | Synthetic |
## Citation
```bibtex
@misc{echo-ultimate-2025,
title = {ECHO ULTIMATE: Metacognitive Calibration RL Environment},
author = {Tripathi, Revtiraman and Pandey, Vikas Dev},
year = {2025},
url = {https://huggingface.co/spaces/Vikaspandey582003/echo-ultimate},
note = {OpenEnv Hackathon 2025}
}
```
"""
# Dockerfile written into the Space — uses Python 3.11 to avoid audioop/pydub issue
_SPACE_DOCKERFILE = """\
FROM python:3.11-slim
WORKDIR /app
RUN apt-get update && apt-get install -y --no-install-recommends \\
build-essential curl git && \\
rm -rf /var/lib/apt/lists/*
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY . .
RUN mkdir -p data results/plots
# Pre-generate all plots so Gradio loads instantly (falls back silently on failure)
RUN python scripts/generate_plots.py || echo "Plot pre-generation skipped"
EXPOSE 7860
ENV GRADIO_SERVER_NAME=0.0.0.0
ENV GRADIO_SERVER_PORT=7860
CMD ["python", "app.py"]
"""
_IGNORE = {
"__pycache__", ".git", ".gitignore", "data", "results",
"echo_lora_adapter", "adversarial_questions.json",
".env", "node_modules", ".DS_Store",
}
def _should_skip(p: Path) -> bool:
for part in p.parts:
if part in _IGNORE or part.startswith("."):
return True
return p.suffix == ".pyc"
def build_space_dir(src: Path, dst: Path):
"""Copy project into dst, inject Space README, Dockerfile, and requirements."""
dst.mkdir(parents=True, exist_ok=True)
for item in src.rglob("*"):
rel = item.relative_to(src)
if _should_skip(rel):
continue
target = dst / rel
if item.is_dir():
target.mkdir(parents=True, exist_ok=True)
else:
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(item, target)
# Inject Space-specific files (override project versions)
(dst / "README.md").write_text(_SPACE_README, encoding="utf-8")
(dst / "Dockerfile").write_text(_SPACE_DOCKERFILE, encoding="utf-8")
# Use the lighter space_requirements.txt as requirements.txt
space_req = src / "space_requirements.txt"
if space_req.exists():
shutil.copy2(space_req, dst / "requirements.txt")
print(f" Space dir prepared: {dst}")
return dst
def publish(repo_id: str, token: str, src: Path):
from huggingface_hub import HfApi
api = HfApi(token=token)
print(f"Creating Space: {repo_id} (Docker SDK)")
try:
api.create_repo(
repo_id=repo_id,
repo_type="space",
space_sdk="docker",
exist_ok=True,
private=False,
)
print(" Repo created (or already exists)")
except Exception as exc:
print(f" Note: {exc}")
with tempfile.TemporaryDirectory() as tmp:
space_dir = build_space_dir(src, Path(tmp) / "space")
print("Uploading files to Space…")
api.upload_folder(
folder_path=str(space_dir),
repo_id=repo_id,
repo_type="space",
ignore_patterns=["*.pyc", "__pycache__"],
)
url = f"https://huggingface.co/spaces/{repo_id}"
print(f"\n✅ Space published: {url}")
print(" Docker build takes ~5-10 minutes on HuggingFace.")
return url
def main():
parser = argparse.ArgumentParser(description="Publish ECHO ULTIMATE to HuggingFace Spaces.")
parser.add_argument("--token", required=True, help="HuggingFace API write token")
parser.add_argument("--repo", default="Vikaspandey582003/echo-ultimate",
help="Space repo ID (default: Vikaspandey582003/echo-ultimate)")
args, _ = parser.parse_known_args()
src = Path(__file__).parent.parent.resolve()
publish(args.repo, args.token, src)
if __name__ == "__main__":
main()