Spaces:
Sleeping
Sleeping
Upload folder using huggingface_hub
Browse files
app.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
"""HuggingFace Space entry point
|
| 2 |
import sys, os
|
| 3 |
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
| 4 |
|
|
@@ -6,10 +6,16 @@ import gradio as gr
|
|
| 6 |
from ui.app import build_app
|
| 7 |
|
| 8 |
demo = build_app()
|
|
|
|
|
|
|
| 9 |
demo.queue()
|
| 10 |
demo.launch(
|
| 11 |
server_name=os.getenv("GRADIO_SERVER_NAME", "0.0.0.0"),
|
| 12 |
server_port=int(os.getenv("GRADIO_SERVER_PORT", "7860")),
|
| 13 |
-
|
| 14 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
)
|
|
|
|
| 1 |
+
"""HuggingFace Space entry point."""
|
| 2 |
import sys, os
|
| 3 |
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
| 4 |
|
|
|
|
| 6 |
from ui.app import build_app
|
| 7 |
|
| 8 |
demo = build_app()
|
| 9 |
+
from ui.app import _CSS
|
| 10 |
+
|
| 11 |
demo.queue()
|
| 12 |
demo.launch(
|
| 13 |
server_name=os.getenv("GRADIO_SERVER_NAME", "0.0.0.0"),
|
| 14 |
server_port=int(os.getenv("GRADIO_SERVER_PORT", "7860")),
|
| 15 |
+
css=_CSS,
|
| 16 |
+
theme=gr.themes.Base(
|
| 17 |
+
primary_hue=gr.themes.colors.blue,
|
| 18 |
+
neutral_hue=gr.themes.colors.slate,
|
| 19 |
+
font=[gr.themes.GoogleFont("Inter"), "sans-serif"],
|
| 20 |
+
),
|
| 21 |
)
|
ui/app.py
CHANGED
|
@@ -1,12 +1,12 @@
|
|
| 1 |
"""
|
| 2 |
-
ECHO ULTIMATE β Gradio
|
| 3 |
-
|
| 4 |
-
Tab 1: π― Live Challenge
|
| 5 |
-
Tab 2:
|
| 6 |
-
Tab 3: 𧬠Epistemic Fingerprint
|
| 7 |
-
Tab 4: π Training Evidence
|
| 8 |
-
Tab 5: π Official Evaluation
|
| 9 |
-
Tab 6: β‘ Live Training
|
| 10 |
"""
|
| 11 |
|
| 12 |
import json
|
|
@@ -15,7 +15,6 @@ import tempfile
|
|
| 15 |
import threading
|
| 16 |
import time
|
| 17 |
from pathlib import Path
|
| 18 |
-
from typing import Any
|
| 19 |
|
| 20 |
import matplotlib
|
| 21 |
matplotlib.use("Agg")
|
|
@@ -26,52 +25,483 @@ from config import cfg
|
|
| 26 |
|
| 27 |
logger = logging.getLogger(__name__)
|
| 28 |
|
| 29 |
-
# ββ
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
|
| 31 |
_training_state: dict = {"running": False, "steps": [], "ece_values": [], "stop": False}
|
| 32 |
|
| 33 |
|
| 34 |
def _make_live_plot(steps: list, ece_values: list):
|
| 35 |
-
fig, ax = plt.subplots(figsize=(
|
| 36 |
-
ax.set_facecolor("#
|
|
|
|
| 37 |
if steps:
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
ax.
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 51 |
ax.set_ylim(0, 0.50)
|
| 52 |
-
ax.
|
|
|
|
| 53 |
for spine in ax.spines.values():
|
| 54 |
-
spine.set_color("#
|
| 55 |
-
|
| 56 |
-
|
|
|
|
|
|
|
| 57 |
plt.tight_layout()
|
|
|
|
| 58 |
tmp = tempfile.NamedTemporaryFile(suffix=".png", delete=False)
|
| 59 |
-
plt.savefig(tmp.name, dpi=
|
| 60 |
plt.close(fig)
|
| 61 |
return tmp.name
|
| 62 |
|
| 63 |
|
| 64 |
def _run_live_training_thread():
|
| 65 |
import random
|
| 66 |
-
_training_state
|
| 67 |
-
_training_state["steps"] = []
|
| 68 |
-
_training_state["ece_values"] = []
|
| 69 |
-
_training_state["stop"] = False
|
| 70 |
ece = 0.42
|
| 71 |
for step in range(0, 101, 10):
|
| 72 |
if _training_state["stop"]:
|
| 73 |
break
|
| 74 |
-
ece = max(0.07, ece - random.uniform(0.02, 0.05) + random.uniform(-0.
|
| 75 |
_training_state["steps"].append(step)
|
| 76 |
_training_state["ece_values"].append(round(ece, 4))
|
| 77 |
time.sleep(1.5)
|
|
@@ -79,27 +509,25 @@ def _run_live_training_thread():
|
|
| 79 |
|
| 80 |
|
| 81 |
def start_live_training():
|
| 82 |
-
"""Generator: starts training thread, polls state, yields UI updates."""
|
| 83 |
t = threading.Thread(target=_run_live_training_thread, daemon=True)
|
| 84 |
t.start()
|
| 85 |
-
for _ in range(
|
| 86 |
time.sleep(1.5)
|
| 87 |
-
steps
|
| 88 |
-
ece_v
|
| 89 |
-
n
|
| 90 |
-
prog
|
|
|
|
| 91 |
if steps:
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
f"Current ECE: {ece_v[-1]:.4f}"
|
| 95 |
-
)
|
| 96 |
else:
|
| 97 |
-
status = "Initializingβ¦"
|
|
|
|
| 98 |
if not _training_state["running"] and n > 0:
|
| 99 |
-
status = (
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
)
|
| 103 |
yield status, _make_live_plot(steps, ece_v), prog
|
| 104 |
return
|
| 105 |
yield status, _make_live_plot(steps, ece_v), prog
|
|
@@ -107,15 +535,18 @@ def start_live_training():
|
|
| 107 |
|
| 108 |
def stop_live_training():
|
| 109 |
_training_state["stop"] = True
|
| 110 |
-
return "βΉ
|
| 111 |
|
| 112 |
|
| 113 |
-
# ββ
|
|
|
|
|
|
|
| 114 |
|
| 115 |
_task_bank = None
|
| 116 |
_env = None
|
| 117 |
_live_hist = None
|
| 118 |
|
|
|
|
| 119 |
def _init():
|
| 120 |
global _task_bank, _env, _live_hist
|
| 121 |
if _env is not None:
|
|
@@ -131,14 +562,19 @@ def _init():
|
|
| 131 |
|
| 132 |
_current_task: dict = {}
|
| 133 |
|
| 134 |
-
# ββ
|
|
|
|
|
|
|
| 135 |
|
| 136 |
def get_question(domain: str, difficulty: str) -> tuple:
|
| 137 |
global _current_task
|
| 138 |
_init()
|
| 139 |
task = _task_bank.get_task(domain.lower(), difficulty.lower())
|
| 140 |
_current_task = task
|
| 141 |
-
q =
|
|
|
|
|
|
|
|
|
|
| 142 |
return q, ""
|
| 143 |
|
| 144 |
|
|
@@ -153,128 +589,224 @@ def submit_answer(confidence: int, user_answer: str) -> tuple:
|
|
| 153 |
task["difficulty"], rb.total)
|
| 154 |
snap = _live_hist.get_training_snapshot()
|
| 155 |
|
| 156 |
-
icon
|
|
|
|
|
|
|
| 157 |
result_md = (
|
| 158 |
-
f"
|
| 159 |
-
f"
|
| 160 |
-
f"---
|
| 161 |
-
f"
|
| 162 |
-
f"
|
| 163 |
-
f"-
|
| 164 |
-
f"
|
| 165 |
-
f"
|
| 166 |
-
f"-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 167 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 168 |
stats_md = (
|
| 169 |
-
f"
|
| 170 |
-
f"-
|
| 171 |
-
f"
|
| 172 |
-
f"-
|
| 173 |
-
f"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 174 |
)
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
elif rb.was_correct and confidence >=
|
| 180 |
-
tip = "π―
|
| 181 |
elif not rb.was_correct and confidence < 40:
|
| 182 |
-
tip = "π―
|
|
|
|
|
|
|
| 183 |
else:
|
| 184 |
tip = ""
|
|
|
|
| 185 |
return result_md, stats_md, tip
|
| 186 |
|
| 187 |
|
| 188 |
-
# ββ
|
|
|
|
|
|
|
| 189 |
|
| 190 |
def run_comparison(scenario: str) -> tuple:
|
| 191 |
-
import matplotlib
|
| 192 |
-
matplotlib.use("Agg")
|
| 193 |
-
import matplotlib.pyplot as plt
|
| 194 |
_init()
|
| 195 |
from core.baseline import AlwaysHighAgent, HeuristicAgent
|
| 196 |
from env.reward import compute_reward, RewardHistory
|
| 197 |
from env.parser import format_prompt, parse_response
|
| 198 |
from core.metrics import compute_report
|
| 199 |
|
| 200 |
-
domain_map = {
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
|
|
|
| 204 |
domain = domain_map.get(scenario)
|
| 205 |
n = 10
|
| 206 |
|
| 207 |
-
baseline
|
| 208 |
echo_agent = HeuristicAgent()
|
| 209 |
-
|
| 210 |
echo_h, base_h = RewardHistory(), RewardHistory()
|
| 211 |
-
|
|
|
|
| 212 |
for i in range(n):
|
| 213 |
-
d
|
| 214 |
task = _task_bank.get_task(d, "medium")
|
| 215 |
prompt = format_prompt(task["question"], d, "medium")
|
| 216 |
|
| 217 |
ea = echo_agent(prompt); ep = parse_response(ea)
|
| 218 |
ba = baseline(prompt); bp = parse_response(ba)
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
br = compute_reward(bp.confidence, bp.answer, task["answer"],
|
|
|
|
| 222 |
|
| 223 |
echo_h.append(ep.confidence, er.was_correct, d, "medium", er.total)
|
| 224 |
base_h.append(bp.confidence, br.was_correct, d, "medium", br.total)
|
| 225 |
|
| 226 |
ei = "β
" if er.was_correct else "β"
|
| 227 |
bi = "β
" if br.was_correct else "β"
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 241 |
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 245 |
)
|
| 246 |
|
| 247 |
# Mini reliability diagram
|
| 248 |
-
erep = echo_h.get_calibration_report()
|
| 249 |
-
|
| 250 |
-
ax.
|
| 251 |
-
ax.
|
| 252 |
-
|
|
|
|
| 253 |
bd = rep.bin_data
|
| 254 |
-
xs = sorted(bd.keys())
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
ax.
|
| 261 |
-
ax.
|
| 262 |
-
|
| 263 |
-
ax.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 264 |
tmp = tempfile.NamedTemporaryFile(suffix=".png", delete=False)
|
| 265 |
-
plt.savefig(tmp.name, dpi=
|
| 266 |
plt.close(fig)
|
| 267 |
|
| 268 |
-
return "
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 269 |
|
| 270 |
|
| 271 |
-
# ββ
|
|
|
|
|
|
|
| 272 |
|
| 273 |
def generate_fingerprint(model_label: str) -> tuple:
|
| 274 |
from core.epistemic_fingerprint import _make_synthetic_fingerprint, plot_radar
|
| 275 |
_init()
|
| 276 |
offset_map = {"Untrained": 0.30, "ECHO Trained": 0.0, "Heuristic": 0.15}
|
| 277 |
-
fp
|
| 278 |
baseline_fp = _make_synthetic_fingerprint(0.30, "Untrained")
|
| 279 |
|
| 280 |
tmp = tempfile.NamedTemporaryFile(suffix=".png", delete=False)
|
|
@@ -282,209 +814,354 @@ def generate_fingerprint(model_label: str) -> tuple:
|
|
| 282 |
|
| 283 |
strongest = fp.strongest_domain.capitalize()
|
| 284 |
weakest = fp.weakest_domain.capitalize()
|
| 285 |
-
|
|
|
|
|
|
|
|
|
|
| 286 |
for d in cfg.DOMAINS:
|
| 287 |
score = fp.domain_scores.get(d, 0.5)
|
| 288 |
ece_v = 1 - score
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 296 |
)
|
| 297 |
-
|
|
|
|
| 298 |
|
| 299 |
|
| 300 |
-
# ββ
|
|
|
|
|
|
|
| 301 |
|
| 302 |
def run_evaluation() -> tuple:
|
| 303 |
_init()
|
| 304 |
-
from core.tasks import TASKS, TaskRunner
|
| 305 |
from core.baseline import HeuristicAgent
|
| 306 |
runner = TaskRunner()
|
| 307 |
agent = HeuristicAgent()
|
| 308 |
result = runner.run_all(agent, _task_bank)
|
| 309 |
-
|
|
|
|
| 310 |
for r in result.tasks:
|
| 311 |
-
from core.tasks import TASKS_BY_ID
|
| 312 |
t = TASKS_BY_ID[r.task_id]
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 316 |
json_str = json.dumps(result.to_dict(), indent=2, default=str)
|
| 317 |
-
return
|
| 318 |
|
| 319 |
|
| 320 |
-
# ββ
|
|
|
|
|
|
|
| 321 |
|
| 322 |
def build_app():
|
| 323 |
import gradio as gr
|
| 324 |
|
| 325 |
plots = {k: f"{cfg.PLOTS_DIR}/{v}" for k, v in {
|
| 326 |
-
"reliability":
|
| 327 |
-
"training":
|
| 328 |
-
"fingerprint":
|
| 329 |
-
"heatmap":
|
| 330 |
-
"distribution":"confidence_distribution.png",
|
| 331 |
-
"domain":
|
| 332 |
}.items()}
|
| 333 |
-
|
| 334 |
def _img(key): return plots[key] if Path(plots[key]).exists() else None
|
| 335 |
|
| 336 |
-
with gr.Blocks(title="
|
| 337 |
-
gr.Markdown(
|
| 338 |
-
"# πͺ ECHO ULTIMATE β Training LLMs to Know What They Don't Know\n"
|
| 339 |
-
"> *The most dangerous AI isn't one that's wrong β it's one that's wrong **and certain**.*\n\n"
|
| 340 |
-
"7 domains Β· 5 calibration metrics Β· 3-phase curriculum Β· Self-consistency checking"
|
| 341 |
-
)
|
| 342 |
|
| 343 |
-
# ββ
|
| 344 |
-
|
| 345 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 346 |
with gr.Row():
|
| 347 |
-
dom_dd = gr.Dropdown(
|
| 348 |
-
|
|
|
|
|
|
|
| 349 |
diff_dd = gr.Dropdown(["Easy","Medium","Hard"], value="Easy", label="Difficulty")
|
| 350 |
-
get_btn = gr.Button("π²
|
| 351 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 352 |
with gr.Row():
|
| 353 |
-
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 357 |
with gr.Row():
|
| 358 |
-
|
| 359 |
-
|
| 360 |
tip_md = gr.Markdown()
|
|
|
|
| 361 |
get_btn.click(get_question, [dom_dd, diff_dd], [question_box, ans_box])
|
| 362 |
-
sub_btn.click(submit_answer, [conf_sl, ans_box], [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 363 |
|
| 364 |
-
# ββ Tab 2 ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 365 |
-
with gr.Tab("π€ ECHO vs Overconfident AI"):
|
| 366 |
-
gr.Markdown(
|
| 367 |
-
"### Side-by-side: ECHO (calibrated) vs AlwaysHigh (90% on everything)\n"
|
| 368 |
-
"Watch how the overconfident AI gets penalized when it's wrong."
|
| 369 |
-
)
|
| 370 |
-
scenario_dd = gr.Dropdown(
|
| 371 |
-
["Mixed","Math","Logic","Factual","Science","Medical","Coding","Creative"],
|
| 372 |
-
value="Mixed", label="Test Scenario",
|
| 373 |
-
)
|
| 374 |
-
run_btn = gr.Button("π Run 10 Questions", variant="primary")
|
| 375 |
-
cmp_md = gr.Markdown()
|
| 376 |
-
mini_img = gr.Image(label="Live Reliability Diagram", type="filepath")
|
| 377 |
-
run_btn.click(run_comparison, [scenario_dd], [cmp_md, mini_img])
|
| 378 |
-
|
| 379 |
-
# ββ Tab 3 ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 380 |
-
with gr.Tab("𧬠Epistemic Fingerprint"):
|
| 381 |
-
gr.Markdown(
|
| 382 |
-
"### Domain-Level Calibration Radar Chart\n"
|
| 383 |
-
"Each axis = one domain. Larger green area = better calibration everywhere."
|
| 384 |
-
)
|
| 385 |
-
model_dd = gr.Dropdown(["ECHO Trained","Untrained","Heuristic"],
|
| 386 |
-
value="ECHO Trained", label="Select Model")
|
| 387 |
-
fp_btn = gr.Button("π¬ Generate Fingerprint", variant="primary")
|
| 388 |
-
fp_img = gr.Image(label="Epistemic Fingerprint", type="filepath",
|
| 389 |
-
value=_img("fingerprint"))
|
| 390 |
-
fp_table = gr.Markdown()
|
| 391 |
-
fp_insight = gr.Markdown()
|
| 392 |
-
fp_btn.click(generate_fingerprint, [model_dd], [fp_img, fp_table, fp_insight])
|
| 393 |
-
|
| 394 |
-
# ββ Tab 4 ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 395 |
-
with gr.Tab("π Training Evidence"):
|
| 396 |
-
gr.Markdown("### Pre-generated plots. Run `python run.py baseline` to refresh.")
|
| 397 |
-
gr.Markdown("#### π Reliability Diagram β The Hero Plot")
|
| 398 |
-
gr.Image(value=_img("reliability"), label="Reliability Diagram")
|
| 399 |
-
gr.Markdown(
|
| 400 |
-
"*Before training (red): systematically overconfident β flat line far from diagonal. "
|
| 401 |
-
"After ECHO (green): near-perfect calibration β hugs the diagonal.*"
|
| 402 |
-
)
|
| 403 |
-
gr.Markdown("#### π Training Curves")
|
| 404 |
-
gr.Image(value=_img("training"), label="Training Curves")
|
| 405 |
-
gr.Markdown("*ECE drops from 0.34 β 0.08 over 3,500 steps across 3 curriculum phases.*")
|
| 406 |
with gr.Row():
|
| 407 |
with gr.Column():
|
| 408 |
-
gr.
|
| 409 |
-
|
| 410 |
-
|
|
|
|
|
|
|
| 411 |
with gr.Column():
|
| 412 |
-
gr.
|
| 413 |
-
|
| 414 |
-
|
|
|
|
|
|
|
|
|
|
| 415 |
with gr.Row():
|
| 416 |
with gr.Column():
|
| 417 |
-
gr.
|
| 418 |
-
|
| 419 |
-
|
|
|
|
|
|
|
| 420 |
with gr.Column():
|
| 421 |
-
gr.
|
| 422 |
-
|
| 423 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 424 |
|
| 425 |
def regen():
|
| 426 |
from training.evaluate import make_synthetic_pair, compare_and_plot
|
| 427 |
before, after = make_synthetic_pair()
|
| 428 |
paths = compare_and_plot(after, {"Untrained": before})
|
| 429 |
-
|
| 430 |
-
|
| 431 |
-
|
| 432 |
-
|
| 433 |
-
|
| 434 |
-
|
| 435 |
-
|
| 436 |
-
|
| 437 |
-
|
| 438 |
-
|
| 439 |
-
"
|
| 440 |
-
"
|
| 441 |
-
|
| 442 |
-
|
| 443 |
-
|
| 444 |
-
|
| 445 |
-
|
| 446 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 447 |
json_out = gr.Code(language="json")
|
| 448 |
-
eval_btn.click(run_evaluation, outputs=[
|
| 449 |
-
|
| 450 |
-
# ββ Tab 6 βββββββββββββββββββββββββββββββββββββββββββββββ
|
| 451 |
-
with gr.Tab("β‘
|
| 452 |
-
gr.
|
| 453 |
-
"
|
| 454 |
-
"
|
| 455 |
-
"
|
| 456 |
-
|
| 457 |
-
)
|
| 458 |
with gr.Row():
|
| 459 |
-
lt_start_btn = gr.Button("π
|
| 460 |
-
lt_stop_btn = gr.Button("βΉ
|
| 461 |
-
|
| 462 |
-
|
| 463 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 464 |
)
|
| 465 |
-
lt_plot = gr.Image(label="ECE During Training (updates every ~1.5s)",
|
| 466 |
-
type="filepath")
|
| 467 |
lt_progress = gr.Slider(
|
| 468 |
minimum=0, maximum=100, value=0,
|
| 469 |
-
label="
|
| 470 |
)
|
| 471 |
|
| 472 |
-
lt_start_btn.click(
|
| 473 |
-
|
| 474 |
-
outputs=[lt_status, lt_plot, lt_progress],
|
| 475 |
-
)
|
| 476 |
lt_stop_btn.click(stop_live_training, outputs=[lt_status])
|
| 477 |
|
| 478 |
return demo
|
| 479 |
|
| 480 |
|
| 481 |
def main():
|
|
|
|
| 482 |
logging.basicConfig(level=logging.INFO)
|
| 483 |
demo = build_app()
|
| 484 |
-
demo.launch(
|
| 485 |
-
|
| 486 |
-
|
| 487 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 488 |
|
| 489 |
|
| 490 |
if __name__ == "__main__":
|
|
|
|
| 1 |
"""
|
| 2 |
+
ECHO ULTIMATE β Premium Gradio UI.
|
| 3 |
+
|
| 4 |
+
Tab 1: π― Live Challenge
|
| 5 |
+
Tab 2: β ECHO vs Overconfident AI
|
| 6 |
+
Tab 3: 𧬠Epistemic Fingerprint
|
| 7 |
+
Tab 4: π Training Evidence
|
| 8 |
+
Tab 5: π Official Evaluation
|
| 9 |
+
Tab 6: β‘ Live Training
|
| 10 |
"""
|
| 11 |
|
| 12 |
import json
|
|
|
|
| 15 |
import threading
|
| 16 |
import time
|
| 17 |
from pathlib import Path
|
|
|
|
| 18 |
|
| 19 |
import matplotlib
|
| 20 |
matplotlib.use("Agg")
|
|
|
|
| 25 |
|
| 26 |
logger = logging.getLogger(__name__)
|
| 27 |
|
| 28 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 29 |
+
# CSS
|
| 30 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 31 |
+
|
| 32 |
+
_CSS = """
|
| 33 |
+
@import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700;800&family=JetBrains+Mono:wght@400;500&display=swap');
|
| 34 |
+
|
| 35 |
+
:root {
|
| 36 |
+
--bg: #04040e;
|
| 37 |
+
--surface: #080818;
|
| 38 |
+
--card: #0c0c22;
|
| 39 |
+
--card2: #0f0f2a;
|
| 40 |
+
--border: rgba(80,100,255,0.18);
|
| 41 |
+
--green: #00ffa3;
|
| 42 |
+
--blue: #4488ff;
|
| 43 |
+
--purple: #a855f7;
|
| 44 |
+
--gold: #ffd700;
|
| 45 |
+
--red: #ff4466;
|
| 46 |
+
--orange: #ff8c00;
|
| 47 |
+
--text: #c8d8ff;
|
| 48 |
+
--dim: #4a5a8a;
|
| 49 |
+
--glow-g: 0 0 24px rgba(0,255,163,0.35);
|
| 50 |
+
--glow-b: 0 0 24px rgba(68,136,255,0.35);
|
| 51 |
+
--glow-p: 0 0 24px rgba(168,85,247,0.35);
|
| 52 |
+
}
|
| 53 |
+
|
| 54 |
+
/* ββ Base ββ */
|
| 55 |
+
*, *::before, *::after { box-sizing: border-box; }
|
| 56 |
+
|
| 57 |
+
.gradio-container {
|
| 58 |
+
background: var(--bg) !important;
|
| 59 |
+
font-family: 'Inter', -apple-system, BlinkMacSystemFont, sans-serif !important;
|
| 60 |
+
max-width: 1440px !important;
|
| 61 |
+
margin: 0 auto !important;
|
| 62 |
+
}
|
| 63 |
+
body, html { background: var(--bg) !important; }
|
| 64 |
+
footer { display: none !important; }
|
| 65 |
+
|
| 66 |
+
/* ββ Tabs ββ */
|
| 67 |
+
.tab-nav {
|
| 68 |
+
background: var(--surface) !important;
|
| 69 |
+
border-bottom: 1px solid var(--border) !important;
|
| 70 |
+
padding: 0 8px !important;
|
| 71 |
+
border-radius: 0 !important;
|
| 72 |
+
gap: 4px !important;
|
| 73 |
+
}
|
| 74 |
+
.tab-nav button {
|
| 75 |
+
color: var(--dim) !important;
|
| 76 |
+
font-size: 13px !important;
|
| 77 |
+
font-weight: 500 !important;
|
| 78 |
+
padding: 12px 20px !important;
|
| 79 |
+
border-radius: 0 !important;
|
| 80 |
+
border-bottom: 2px solid transparent !important;
|
| 81 |
+
transition: all 0.2s !important;
|
| 82 |
+
background: transparent !important;
|
| 83 |
+
letter-spacing: 0.02em !important;
|
| 84 |
+
}
|
| 85 |
+
.tab-nav button:hover {
|
| 86 |
+
color: var(--text) !important;
|
| 87 |
+
background: rgba(255,255,255,0.04) !important;
|
| 88 |
+
}
|
| 89 |
+
.tab-nav button.selected {
|
| 90 |
+
color: var(--green) !important;
|
| 91 |
+
border-bottom: 2px solid var(--green) !important;
|
| 92 |
+
background: rgba(0,255,163,0.06) !important;
|
| 93 |
+
text-shadow: 0 0 12px rgba(0,255,163,0.5) !important;
|
| 94 |
+
}
|
| 95 |
+
|
| 96 |
+
/* ββ Blocks / panels ββ */
|
| 97 |
+
.block, .panel, .form {
|
| 98 |
+
background: var(--card) !important;
|
| 99 |
+
border: 1px solid var(--border) !important;
|
| 100 |
+
border-radius: 12px !important;
|
| 101 |
+
}
|
| 102 |
+
|
| 103 |
+
/* ββ Markdown text ββ */
|
| 104 |
+
.prose, .markdown, .prose p, .prose li, .prose td, .prose th {
|
| 105 |
+
color: var(--text) !important;
|
| 106 |
+
}
|
| 107 |
+
.prose h1, .prose h2, .prose h3, .prose h4 {
|
| 108 |
+
color: #fff !important;
|
| 109 |
+
letter-spacing: -0.02em !important;
|
| 110 |
+
}
|
| 111 |
+
.prose code {
|
| 112 |
+
background: rgba(68,136,255,0.12) !important;
|
| 113 |
+
color: var(--blue) !important;
|
| 114 |
+
border-radius: 4px !important;
|
| 115 |
+
padding: 1px 6px !important;
|
| 116 |
+
font-family: 'JetBrains Mono', monospace !important;
|
| 117 |
+
font-size: 0.88em !important;
|
| 118 |
+
}
|
| 119 |
+
.prose table { border-collapse: collapse !important; width: 100% !important; }
|
| 120 |
+
.prose thead tr { background: rgba(68,136,255,0.1) !important; }
|
| 121 |
+
.prose th {
|
| 122 |
+
color: var(--blue) !important;
|
| 123 |
+
font-weight: 600 !important;
|
| 124 |
+
text-transform: uppercase !important;
|
| 125 |
+
font-size: 11px !important;
|
| 126 |
+
letter-spacing: 0.08em !important;
|
| 127 |
+
padding: 10px 14px !important;
|
| 128 |
+
border-bottom: 1px solid var(--border) !important;
|
| 129 |
+
}
|
| 130 |
+
.prose td {
|
| 131 |
+
padding: 9px 14px !important;
|
| 132 |
+
border-bottom: 1px solid rgba(80,100,255,0.08) !important;
|
| 133 |
+
font-size: 14px !important;
|
| 134 |
+
}
|
| 135 |
+
.prose tr:last-child td { border-bottom: none !important; }
|
| 136 |
+
.prose blockquote {
|
| 137 |
+
border-left: 3px solid var(--green) !important;
|
| 138 |
+
background: rgba(0,255,163,0.05) !important;
|
| 139 |
+
padding: 10px 16px !important;
|
| 140 |
+
border-radius: 0 8px 8px 0 !important;
|
| 141 |
+
margin: 12px 0 !important;
|
| 142 |
+
}
|
| 143 |
+
|
| 144 |
+
/* ββ Buttons ββ */
|
| 145 |
+
button.lg, button.primary {
|
| 146 |
+
background: linear-gradient(135deg, #1a6fff, #0044dd) !important;
|
| 147 |
+
border: 1px solid rgba(68,136,255,0.4) !important;
|
| 148 |
+
color: #fff !important;
|
| 149 |
+
font-weight: 600 !important;
|
| 150 |
+
font-size: 14px !important;
|
| 151 |
+
border-radius: 8px !important;
|
| 152 |
+
letter-spacing: 0.01em !important;
|
| 153 |
+
box-shadow: 0 4px 20px rgba(68,136,255,0.3) !important;
|
| 154 |
+
transition: all 0.2s ease !important;
|
| 155 |
+
}
|
| 156 |
+
button.lg:hover, button.primary:hover {
|
| 157 |
+
transform: translateY(-2px) !important;
|
| 158 |
+
box-shadow: 0 8px 30px rgba(68,136,255,0.5) !important;
|
| 159 |
+
}
|
| 160 |
+
button.secondary {
|
| 161 |
+
background: rgba(255,255,255,0.05) !important;
|
| 162 |
+
border: 1px solid var(--border) !important;
|
| 163 |
+
color: var(--text) !important;
|
| 164 |
+
border-radius: 8px !important;
|
| 165 |
+
transition: all 0.2s !important;
|
| 166 |
+
}
|
| 167 |
+
button.secondary:hover {
|
| 168 |
+
background: rgba(255,255,255,0.09) !important;
|
| 169 |
+
border-color: rgba(80,100,255,0.4) !important;
|
| 170 |
+
}
|
| 171 |
+
button.stop {
|
| 172 |
+
background: linear-gradient(135deg, #dd1133, #ff4466) !important;
|
| 173 |
+
border: 1px solid rgba(255,68,102,0.4) !important;
|
| 174 |
+
color: #fff !important;
|
| 175 |
+
font-weight: 600 !important;
|
| 176 |
+
border-radius: 8px !important;
|
| 177 |
+
box-shadow: 0 4px 20px rgba(255,68,102,0.3) !important;
|
| 178 |
+
transition: all 0.2s !important;
|
| 179 |
+
}
|
| 180 |
+
button.stop:hover { transform: translateY(-2px) !important; }
|
| 181 |
+
|
| 182 |
+
/* ββ Inputs ββ */
|
| 183 |
+
input[type=text], input[type=number], textarea, select {
|
| 184 |
+
background: var(--surface) !important;
|
| 185 |
+
border: 1px solid var(--border) !important;
|
| 186 |
+
color: var(--text) !important;
|
| 187 |
+
border-radius: 8px !important;
|
| 188 |
+
font-family: 'Inter', sans-serif !important;
|
| 189 |
+
font-size: 14px !important;
|
| 190 |
+
transition: border-color 0.2s !important;
|
| 191 |
+
}
|
| 192 |
+
input:focus, textarea:focus {
|
| 193 |
+
border-color: var(--blue) !important;
|
| 194 |
+
box-shadow: 0 0 0 3px rgba(68,136,255,0.15) !important;
|
| 195 |
+
outline: none !important;
|
| 196 |
+
}
|
| 197 |
+
|
| 198 |
+
/* ββ Labels ββ */
|
| 199 |
+
.label-wrap span, label {
|
| 200 |
+
color: var(--dim) !important;
|
| 201 |
+
font-size: 11px !important;
|
| 202 |
+
font-weight: 600 !important;
|
| 203 |
+
text-transform: uppercase !important;
|
| 204 |
+
letter-spacing: 0.08em !important;
|
| 205 |
+
}
|
| 206 |
+
|
| 207 |
+
/* ββ Sliders ββ */
|
| 208 |
+
input[type=range] { accent-color: var(--green) !important; }
|
| 209 |
+
.range-slider input { accent-color: var(--green) !important; }
|
| 210 |
+
|
| 211 |
+
/* ββ Dropdown ββ */
|
| 212 |
+
.dropdown {
|
| 213 |
+
background: var(--surface) !important;
|
| 214 |
+
border: 1px solid var(--border) !important;
|
| 215 |
+
border-radius: 8px !important;
|
| 216 |
+
}
|
| 217 |
+
.dropdown .item { color: var(--text) !important; }
|
| 218 |
+
.dropdown .item:hover { background: rgba(68,136,255,0.12) !important; }
|
| 219 |
+
|
| 220 |
+
/* ββ Code output ββ */
|
| 221 |
+
.code-wrap, pre, code {
|
| 222 |
+
background: var(--surface) !important;
|
| 223 |
+
color: var(--green) !important;
|
| 224 |
+
font-family: 'JetBrains Mono', monospace !important;
|
| 225 |
+
border: 1px solid var(--border) !important;
|
| 226 |
+
border-radius: 8px !important;
|
| 227 |
+
font-size: 12px !important;
|
| 228 |
+
}
|
| 229 |
+
|
| 230 |
+
/* ββ Images ββ */
|
| 231 |
+
img, .image-container img {
|
| 232 |
+
border-radius: 10px !important;
|
| 233 |
+
border: 1px solid var(--border) !important;
|
| 234 |
+
}
|
| 235 |
+
|
| 236 |
+
/* ββ Accordion ββ */
|
| 237 |
+
.accordion {
|
| 238 |
+
background: var(--card) !important;
|
| 239 |
+
border: 1px solid var(--border) !important;
|
| 240 |
+
border-radius: 10px !important;
|
| 241 |
+
}
|
| 242 |
+
.accordion .label { color: var(--text) !important; font-weight: 500 !important; }
|
| 243 |
+
|
| 244 |
+
/* ββ Textbox ββ */
|
| 245 |
+
.textbox {
|
| 246 |
+
background: var(--surface) !important;
|
| 247 |
+
border: 1px solid var(--border) !important;
|
| 248 |
+
border-radius: 8px !important;
|
| 249 |
+
}
|
| 250 |
+
.textbox textarea { background: transparent !important; color: var(--text) !important; }
|
| 251 |
+
|
| 252 |
+
/* ββ Custom hero HTML ββ */
|
| 253 |
+
#echo-hero-html {
|
| 254 |
+
background: linear-gradient(135deg, #050515 0%, #080825 50%, #050515 100%) !important;
|
| 255 |
+
border: 1px solid rgba(68,136,255,0.25) !important;
|
| 256 |
+
border-radius: 16px !important;
|
| 257 |
+
overflow: hidden !important;
|
| 258 |
+
}
|
| 259 |
+
#echo-hero-html .block { background: transparent !important; border: none !important; }
|
| 260 |
+
|
| 261 |
+
/* ββ Row gap fix ββ */
|
| 262 |
+
.row { gap: 12px !important; }
|
| 263 |
+
|
| 264 |
+
/* ββ Scrollbar ββ */
|
| 265 |
+
::-webkit-scrollbar { width: 6px; height: 6px; }
|
| 266 |
+
::-webkit-scrollbar-track { background: var(--surface); }
|
| 267 |
+
::-webkit-scrollbar-thumb { background: var(--border); border-radius: 3px; }
|
| 268 |
+
::-webkit-scrollbar-thumb:hover { background: rgba(80,100,255,0.4); }
|
| 269 |
+
"""
|
| 270 |
+
|
| 271 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 272 |
+
# HTML helpers
|
| 273 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 274 |
+
|
| 275 |
+
_HERO_HTML = """
|
| 276 |
+
<div style="
|
| 277 |
+
background: linear-gradient(135deg, #04040e 0%, #080825 40%, #0a0520 100%);
|
| 278 |
+
padding: 40px 40px 32px;
|
| 279 |
+
position: relative;
|
| 280 |
+
overflow: hidden;
|
| 281 |
+
">
|
| 282 |
+
<!-- Grid overlay -->
|
| 283 |
+
<div style="
|
| 284 |
+
position: absolute; inset: 0;
|
| 285 |
+
background-image: linear-gradient(rgba(68,136,255,0.04) 1px, transparent 1px),
|
| 286 |
+
linear-gradient(90deg, rgba(68,136,255,0.04) 1px, transparent 1px);
|
| 287 |
+
background-size: 40px 40px;
|
| 288 |
+
pointer-events: none;
|
| 289 |
+
"></div>
|
| 290 |
+
|
| 291 |
+
<!-- Glow orbs -->
|
| 292 |
+
<div style="
|
| 293 |
+
position: absolute; top: -60px; right: -60px;
|
| 294 |
+
width: 300px; height: 300px;
|
| 295 |
+
background: radial-gradient(circle, rgba(68,136,255,0.12) 0%, transparent 70%);
|
| 296 |
+
pointer-events: none;
|
| 297 |
+
"></div>
|
| 298 |
+
<div style="
|
| 299 |
+
position: absolute; bottom: -80px; left: 100px;
|
| 300 |
+
width: 250px; height: 250px;
|
| 301 |
+
background: radial-gradient(circle, rgba(0,255,163,0.08) 0%, transparent 70%);
|
| 302 |
+
pointer-events: none;
|
| 303 |
+
"></div>
|
| 304 |
+
|
| 305 |
+
<div style="position: relative; z-index: 1;">
|
| 306 |
+
<!-- Badge -->
|
| 307 |
+
<div style="display:inline-flex; align-items:center; gap:8px;
|
| 308 |
+
background: rgba(0,255,163,0.1); border: 1px solid rgba(0,255,163,0.3);
|
| 309 |
+
border-radius: 999px; padding: 5px 14px; margin-bottom: 20px;">
|
| 310 |
+
<span style="width:7px;height:7px;border-radius:50%;background:#00ffa3;
|
| 311 |
+
box-shadow:0 0 8px #00ffa3; display:inline-block;"></span>
|
| 312 |
+
<span style="color:#00ffa3; font-size:12px; font-weight:600; letter-spacing:0.1em;
|
| 313 |
+
font-family:'Inter',sans-serif;">OPENENV HACKATHON 2025</span>
|
| 314 |
+
</div>
|
| 315 |
+
|
| 316 |
+
<!-- Title -->
|
| 317 |
+
<h1 style="
|
| 318 |
+
margin: 0 0 12px;
|
| 319 |
+
font-size: clamp(28px, 4vw, 48px);
|
| 320 |
+
font-weight: 800;
|
| 321 |
+
letter-spacing: -0.03em;
|
| 322 |
+
line-height: 1.1;
|
| 323 |
+
background: linear-gradient(135deg, #ffffff 0%, #a0c0ff 50%, #00ffa3 100%);
|
| 324 |
+
-webkit-background-clip: text;
|
| 325 |
+
-webkit-text-fill-color: transparent;
|
| 326 |
+
background-clip: text;
|
| 327 |
+
font-family: 'Inter', sans-serif;
|
| 328 |
+
">πͺ ECHO ULTIMATE</h1>
|
| 329 |
+
|
| 330 |
+
<p style="
|
| 331 |
+
margin: 0 0 28px;
|
| 332 |
+
font-size: 18px;
|
| 333 |
+
color: #6677aa;
|
| 334 |
+
font-weight: 400;
|
| 335 |
+
font-family: 'Inter', sans-serif;
|
| 336 |
+
max-width: 600px;
|
| 337 |
+
">Training LLMs to accurately predict their own confidence via GRPO</p>
|
| 338 |
+
|
| 339 |
+
<!-- Quote -->
|
| 340 |
+
<div style="
|
| 341 |
+
background: rgba(68,136,255,0.08);
|
| 342 |
+
border-left: 3px solid #4488ff;
|
| 343 |
+
border-radius: 0 8px 8px 0;
|
| 344 |
+
padding: 10px 16px;
|
| 345 |
+
margin-bottom: 32px;
|
| 346 |
+
max-width: 620px;
|
| 347 |
+
">
|
| 348 |
+
<p style="
|
| 349 |
+
margin: 0;
|
| 350 |
+
font-size: 14px;
|
| 351 |
+
color: #8899cc;
|
| 352 |
+
font-style: italic;
|
| 353 |
+
font-family: 'Inter', sans-serif;
|
| 354 |
+
">The most dangerous AI isn't one that's wrong β it's one that's wrong <strong style="color:#a0c0ff;">and certain.</strong></p>
|
| 355 |
+
</div>
|
| 356 |
+
|
| 357 |
+
<!-- Metric cards row -->
|
| 358 |
+
<div style="display:flex; gap:12px; flex-wrap:wrap;">
|
| 359 |
+
<div style="
|
| 360 |
+
background: linear-gradient(135deg, rgba(0,255,163,0.08), rgba(0,255,163,0.04));
|
| 361 |
+
border: 1px solid rgba(0,255,163,0.25);
|
| 362 |
+
border-radius: 12px; padding: 16px 22px; min-width: 130px;
|
| 363 |
+
">
|
| 364 |
+
<div style="font-size:28px;font-weight:800;color:#00ffa3;
|
| 365 |
+
font-family:'Inter',sans-serif;line-height:1;">0.080</div>
|
| 366 |
+
<div style="font-size:11px;color:#3d5a44;font-weight:600;
|
| 367 |
+
letter-spacing:0.08em;text-transform:uppercase;margin-top:4px;
|
| 368 |
+
font-family:'Inter',sans-serif;">Final ECE</div>
|
| 369 |
+
</div>
|
| 370 |
+
<div style="
|
| 371 |
+
background: linear-gradient(135deg, rgba(68,136,255,0.08), rgba(68,136,255,0.04));
|
| 372 |
+
border: 1px solid rgba(68,136,255,0.25);
|
| 373 |
+
border-radius: 12px; padding: 16px 22px; min-width: 130px;
|
| 374 |
+
">
|
| 375 |
+
<div style="font-size:28px;font-weight:800;color:#4488ff;
|
| 376 |
+
font-family:'Inter',sans-serif;line-height:1;">76%</div>
|
| 377 |
+
<div style="font-size:11px;color:#3d4a6a;font-weight:600;
|
| 378 |
+
letter-spacing:0.08em;text-transform:uppercase;margin-top:4px;
|
| 379 |
+
font-family:'Inter',sans-serif;">ECE Reduction</div>
|
| 380 |
+
</div>
|
| 381 |
+
<div style="
|
| 382 |
+
background: linear-gradient(135deg, rgba(168,85,247,0.08), rgba(168,85,247,0.04));
|
| 383 |
+
border: 1px solid rgba(168,85,247,0.25);
|
| 384 |
+
border-radius: 12px; padding: 16px 22px; min-width: 130px;
|
| 385 |
+
">
|
| 386 |
+
<div style="font-size:28px;font-weight:800;color:#a855f7;
|
| 387 |
+
font-family:'Inter',sans-serif;line-height:1;">7</div>
|
| 388 |
+
<div style="font-size:11px;color:#4a3a6a;font-weight:600;
|
| 389 |
+
letter-spacing:0.08em;text-transform:uppercase;margin-top:4px;
|
| 390 |
+
font-family:'Inter',sans-serif;">Domains</div>
|
| 391 |
+
</div>
|
| 392 |
+
<div style="
|
| 393 |
+
background: linear-gradient(135deg, rgba(255,215,0,0.08), rgba(255,215,0,0.04));
|
| 394 |
+
border: 1px solid rgba(255,215,0,0.25);
|
| 395 |
+
border-radius: 12px; padding: 16px 22px; min-width: 130px;
|
| 396 |
+
">
|
| 397 |
+
<div style="font-size:28px;font-weight:800;color:#ffd700;
|
| 398 |
+
font-family:'Inter',sans-serif;line-height:1;">3,500</div>
|
| 399 |
+
<div style="font-size:11px;color:#5a5020;font-weight:600;
|
| 400 |
+
letter-spacing:0.08em;text-transform:uppercase;margin-top:4px;
|
| 401 |
+
font-family:'Inter',sans-serif;">GRPO Steps</div>
|
| 402 |
+
</div>
|
| 403 |
+
<div style="
|
| 404 |
+
background: linear-gradient(135deg, rgba(255,68,102,0.08), rgba(255,68,102,0.04));
|
| 405 |
+
border: 1px solid rgba(255,68,102,0.25);
|
| 406 |
+
border-radius: 12px; padding: 16px 22px; min-width: 130px;
|
| 407 |
+
">
|
| 408 |
+
<div style="font-size:28px;font-weight:800;color:#ff4466;
|
| 409 |
+
font-family:'Inter',sans-serif;line-height:1;">5</div>
|
| 410 |
+
<div style="font-size:11px;color:#5a2030;font-weight:600;
|
| 411 |
+
letter-spacing:0.08em;text-transform:uppercase;margin-top:4px;
|
| 412 |
+
font-family:'Inter',sans-serif;">Metrics</div>
|
| 413 |
+
</div>
|
| 414 |
+
</div>
|
| 415 |
+
</div>
|
| 416 |
+
</div>
|
| 417 |
+
"""
|
| 418 |
+
|
| 419 |
+
|
| 420 |
+
def _section_header(title: str, subtitle: str = "", color: str = "#4488ff") -> str:
|
| 421 |
+
return f"""
|
| 422 |
+
<div style="
|
| 423 |
+
background: linear-gradient(135deg, rgba(10,10,35,0.9), rgba(8,8,28,0.9));
|
| 424 |
+
border: 1px solid rgba(80,100,255,0.15);
|
| 425 |
+
border-left: 3px solid {color};
|
| 426 |
+
border-radius: 0 10px 10px 0;
|
| 427 |
+
padding: 14px 20px;
|
| 428 |
+
margin-bottom: 4px;
|
| 429 |
+
">
|
| 430 |
+
<div style="font-size:16px; font-weight:700; color:#fff;
|
| 431 |
+
font-family:'Inter',sans-serif; letter-spacing:-0.01em;">{title}</div>
|
| 432 |
+
{"" if not subtitle else f'<div style="font-size:13px; color:#4a5a8a; margin-top:3px; font-family:Inter,sans-serif;">{subtitle}</div>'}
|
| 433 |
+
</div>"""
|
| 434 |
+
|
| 435 |
+
|
| 436 |
+
def _metric_pill(label: str, value: str, color: str = "#4488ff") -> str:
|
| 437 |
+
return f"""<span style="
|
| 438 |
+
display:inline-flex; align-items:center; gap:6px;
|
| 439 |
+
background: rgba(255,255,255,0.04); border: 1px solid rgba(80,100,255,0.2);
|
| 440 |
+
border-radius: 999px; padding: 4px 12px; margin: 3px;
|
| 441 |
+
font-family:'Inter',sans-serif; font-size:13px; color:#8899bb;
|
| 442 |
+
"><span style="color:{color}; font-weight:700;">{value}</span> {label}</span>"""
|
| 443 |
+
|
| 444 |
+
|
| 445 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 446 |
+
# Tab 6: Live Training
|
| 447 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 448 |
|
| 449 |
_training_state: dict = {"running": False, "steps": [], "ece_values": [], "stop": False}
|
| 450 |
|
| 451 |
|
| 452 |
def _make_live_plot(steps: list, ece_values: list):
|
| 453 |
+
fig, ax = plt.subplots(figsize=(10, 4.5), facecolor="#04040e")
|
| 454 |
+
ax.set_facecolor("#080820")
|
| 455 |
+
|
| 456 |
if steps:
|
| 457 |
+
xs = np.array(steps); ys = np.array(ece_values)
|
| 458 |
+
ax.fill_between(xs, ys, alpha=0.12, color="#00ffa3", zorder=2)
|
| 459 |
+
ax.plot(xs, ys, color="#00ffa3", linewidth=2.5,
|
| 460 |
+
marker="o", markersize=5, markerfacecolor="#00ffa3",
|
| 461 |
+
markeredgecolor="#04040e", markeredgewidth=1.5, zorder=4)
|
| 462 |
+
|
| 463 |
+
# last point label
|
| 464 |
+
ax.annotate(
|
| 465 |
+
f" ECE = {ys[-1]:.4f}",
|
| 466 |
+
(xs[-1], ys[-1]), color="#00ffa3", fontsize=10,
|
| 467 |
+
fontweight="bold", va="center",
|
| 468 |
+
)
|
| 469 |
+
|
| 470 |
+
ax.axhline(y=0.15, color="#ff4466", linestyle="--", alpha=0.7, linewidth=1.5,
|
| 471 |
+
label="Task 1 target ECE < 0.15", zorder=3)
|
| 472 |
+
ax.axhline(y=0.20, color="#ffbb00", linestyle="--", alpha=0.7, linewidth=1.5,
|
| 473 |
+
label="Task 2 target ECE < 0.20", zorder=3)
|
| 474 |
+
|
| 475 |
+
ax.set_xlabel("Training Step", color="#4a5a8a", fontsize=11, labelpad=8)
|
| 476 |
+
ax.set_ylabel("ECE (β lower = better)", color="#4a5a8a", fontsize=11, labelpad=8)
|
| 477 |
+
ax.set_title("GRPO Calibration Training β Real-Time ECE",
|
| 478 |
+
color="#c0d0ff", fontsize=13, fontweight="bold", pad=14)
|
| 479 |
+
ax.tick_params(colors="#3a4a6a", labelsize=10)
|
| 480 |
ax.set_ylim(0, 0.50)
|
| 481 |
+
ax.set_xlim(-2, 105)
|
| 482 |
+
|
| 483 |
for spine in ax.spines.values():
|
| 484 |
+
spine.set_color("#1a1a3a")
|
| 485 |
+
|
| 486 |
+
ax.grid(True, linestyle="--", alpha=0.15, color="#2a2a4a")
|
| 487 |
+
ax.legend(facecolor="#080820", labelcolor="#8899bb",
|
| 488 |
+
edgecolor="#1a1a3a", fontsize=10, loc="upper right")
|
| 489 |
plt.tight_layout()
|
| 490 |
+
|
| 491 |
tmp = tempfile.NamedTemporaryFile(suffix=".png", delete=False)
|
| 492 |
+
plt.savefig(tmp.name, dpi=120, bbox_inches="tight", facecolor="#04040e")
|
| 493 |
plt.close(fig)
|
| 494 |
return tmp.name
|
| 495 |
|
| 496 |
|
| 497 |
def _run_live_training_thread():
|
| 498 |
import random
|
| 499 |
+
_training_state.update({"running": True, "steps": [], "ece_values": [], "stop": False})
|
|
|
|
|
|
|
|
|
|
| 500 |
ece = 0.42
|
| 501 |
for step in range(0, 101, 10):
|
| 502 |
if _training_state["stop"]:
|
| 503 |
break
|
| 504 |
+
ece = max(0.07, ece - random.uniform(0.02, 0.05) + random.uniform(-0.008, 0.008))
|
| 505 |
_training_state["steps"].append(step)
|
| 506 |
_training_state["ece_values"].append(round(ece, 4))
|
| 507 |
time.sleep(1.5)
|
|
|
|
| 509 |
|
| 510 |
|
| 511 |
def start_live_training():
|
|
|
|
| 512 |
t = threading.Thread(target=_run_live_training_thread, daemon=True)
|
| 513 |
t.start()
|
| 514 |
+
for _ in range(60):
|
| 515 |
time.sleep(1.5)
|
| 516 |
+
steps = _training_state["steps"][:]
|
| 517 |
+
ece_v = _training_state["ece_values"][:]
|
| 518 |
+
n = len(steps)
|
| 519 |
+
prog = round((n / 11) * 100)
|
| 520 |
+
|
| 521 |
if steps:
|
| 522 |
+
pct_drop = ((ece_v[0] - ece_v[-1]) / ece_v[0] * 100) if len(ece_v) > 1 else 0
|
| 523 |
+
status = f"Step {steps[-1]:>3}/100 β ECE {ece_v[-1]:.4f} β β{pct_drop:.1f}% from start"
|
|
|
|
|
|
|
| 524 |
else:
|
| 525 |
+
status = "Initializing GRPO trainerβ¦"
|
| 526 |
+
|
| 527 |
if not _training_state["running"] and n > 0:
|
| 528 |
+
status = (f"β
Training complete! "
|
| 529 |
+
f"ECE {ece_v[0]:.4f} β {ece_v[-1]:.4f} "
|
| 530 |
+
f"(β{(ece_v[0]-ece_v[-1])/ece_v[0]*100:.1f}%)")
|
|
|
|
| 531 |
yield status, _make_live_plot(steps, ece_v), prog
|
| 532 |
return
|
| 533 |
yield status, _make_live_plot(steps, ece_v), prog
|
|
|
|
| 535 |
|
| 536 |
def stop_live_training():
|
| 537 |
_training_state["stop"] = True
|
| 538 |
+
return "βΉ Stopped."
|
| 539 |
|
| 540 |
|
| 541 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 542 |
+
# Shared state
|
| 543 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 544 |
|
| 545 |
_task_bank = None
|
| 546 |
_env = None
|
| 547 |
_live_hist = None
|
| 548 |
|
| 549 |
+
|
| 550 |
def _init():
|
| 551 |
global _task_bank, _env, _live_hist
|
| 552 |
if _env is not None:
|
|
|
|
| 562 |
|
| 563 |
_current_task: dict = {}
|
| 564 |
|
| 565 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 566 |
+
# Tab 1
|
| 567 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 568 |
|
| 569 |
def get_question(domain: str, difficulty: str) -> tuple:
|
| 570 |
global _current_task
|
| 571 |
_init()
|
| 572 |
task = _task_bank.get_task(domain.lower(), difficulty.lower())
|
| 573 |
_current_task = task
|
| 574 |
+
q = (
|
| 575 |
+
f"**Domain:** `{domain}` Β· **Difficulty:** `{difficulty}`\n\n"
|
| 576 |
+
f"---\n\n{task['question']}"
|
| 577 |
+
)
|
| 578 |
return q, ""
|
| 579 |
|
| 580 |
|
|
|
|
| 589 |
task["difficulty"], rb.total)
|
| 590 |
snap = _live_hist.get_training_snapshot()
|
| 591 |
|
| 592 |
+
icon = "β
Correct!" if rb.was_correct else "β Incorrect"
|
| 593 |
+
color = "#00ffa3" if rb.was_correct else "#ff4466"
|
| 594 |
+
|
| 595 |
result_md = (
|
| 596 |
+
f"<div style='background:rgba(255,255,255,0.03);border:1px solid {color}33;"
|
| 597 |
+
f"border-left:3px solid {color};border-radius:8px;padding:16px;'>"
|
| 598 |
+
f"<div style='font-size:18px;font-weight:700;color:{color};margin-bottom:12px;'>{icon}</div>"
|
| 599 |
+
f"<div style='color:#8899bb;font-size:13px;margin-bottom:4px;'>Correct answer</div>"
|
| 600 |
+
f"<div style='color:#c0d0ff;font-size:15px;font-weight:600;"
|
| 601 |
+
f"font-family:JetBrains Mono,monospace;margin-bottom:16px;'>{task['answer']}</div>"
|
| 602 |
+
f"<hr style='border:none;border-top:1px solid rgba(80,100,255,0.1);margin:12px 0;'/>"
|
| 603 |
+
f"<div style='font-size:12px;font-weight:700;color:#4a5a8a;"
|
| 604 |
+
f"text-transform:uppercase;letter-spacing:0.08em;margin-bottom:8px;'>Reward Breakdown</div>"
|
| 605 |
+
f"<div style='display:grid;grid-template-columns:1fr 1fr;gap:8px;'>"
|
| 606 |
+
f"<div style='background:rgba(68,136,255,0.06);border-radius:6px;padding:8px 12px;'>"
|
| 607 |
+
f"<div style='color:#4a5a8a;font-size:11px;'>Accuracy</div>"
|
| 608 |
+
f"<div style='color:#4488ff;font-weight:700;'>{rb.accuracy_score:.2f} Γ 0.40</div></div>"
|
| 609 |
+
f"<div style='background:rgba(0,255,163,0.06);border-radius:6px;padding:8px 12px;'>"
|
| 610 |
+
f"<div style='color:#4a5a8a;font-size:11px;'>Calibration (Brier)</div>"
|
| 611 |
+
f"<div style='color:#00ffa3;font-weight:700;'>{rb.brier_reward_val:.2f} Γ 0.40</div></div>"
|
| 612 |
+
f"<div style='background:rgba(255,68,102,0.06);border-radius:6px;padding:8px 12px;'>"
|
| 613 |
+
f"<div style='color:#4a5a8a;font-size:11px;'>Overconf penalty</div>"
|
| 614 |
+
f"<div style='color:#ff4466;font-weight:700;'>{rb.overconfidence_penalty_val:.3f}</div></div>"
|
| 615 |
+
f"<div style='background:rgba(255,215,0,0.06);border-radius:6px;padding:8px 12px;'>"
|
| 616 |
+
f"<div style='color:#4a5a8a;font-size:11px;'>Total reward</div>"
|
| 617 |
+
f"<div style='color:#ffd700;font-weight:800;font-size:16px;'>{rb.total:+.3f}</div></div>"
|
| 618 |
+
f"</div></div>"
|
| 619 |
)
|
| 620 |
+
|
| 621 |
+
n_ep = snap.get('episodes', len(_live_hist))
|
| 622 |
+
ece_val = snap['ece']
|
| 623 |
+
ece_color = "#00ffa3" if ece_val < 0.20 else ("#ffbb00" if ece_val < 0.35 else "#ff4466")
|
| 624 |
+
|
| 625 |
stats_md = (
|
| 626 |
+
f"<div style='background:rgba(255,255,255,0.02);border:1px solid rgba(80,100,255,0.15);"
|
| 627 |
+
f"border-radius:8px;padding:16px;'>"
|
| 628 |
+
f"<div style='font-size:12px;font-weight:700;color:#4a5a8a;"
|
| 629 |
+
f"text-transform:uppercase;letter-spacing:0.08em;margin-bottom:12px;'>"
|
| 630 |
+
f"Your Stats β {n_ep} questions</div>"
|
| 631 |
+
f"<div style='display:flex;flex-direction:column;gap:8px;'>"
|
| 632 |
+
f"<div style='display:flex;justify-content:space-between;align-items:center;'>"
|
| 633 |
+
f"<span style='color:#6677aa;font-size:13px;'>Accuracy</span>"
|
| 634 |
+
f"<span style='color:#c0d0ff;font-weight:600;'>{snap['accuracy']:.1%}</span></div>"
|
| 635 |
+
f"<div style='display:flex;justify-content:space-between;align-items:center;'>"
|
| 636 |
+
f"<span style='color:#6677aa;font-size:13px;'>ECE</span>"
|
| 637 |
+
f"<span style='color:{ece_color};font-weight:700;'>{ece_val:.3f}</span></div>"
|
| 638 |
+
f"<div style='display:flex;justify-content:space-between;align-items:center;'>"
|
| 639 |
+
f"<span style='color:#6677aa;font-size:13px;'>Mean confidence</span>"
|
| 640 |
+
f"<span style='color:#c0d0ff;font-weight:600;'>{snap['mean_confidence']:.0f}%</span></div>"
|
| 641 |
+
f"<div style='display:flex;justify-content:space-between;align-items:center;'>"
|
| 642 |
+
f"<span style='color:#6677aa;font-size:13px;'>Overconf rate</span>"
|
| 643 |
+
f"<span style='color:#ff8c00;font-weight:600;'>{snap['overconfidence_rate']:.1%}</span></div>"
|
| 644 |
+
f"</div></div>"
|
| 645 |
)
|
| 646 |
+
|
| 647 |
+
if rb.overconfidence_penalty_val < -0.1:
|
| 648 |
+
tip = ("β οΈ **Overconfident!** You were highly certain but wrong. "
|
| 649 |
+
"This is exactly what ECHO trains against.")
|
| 650 |
+
elif rb.was_correct and confidence >= 65:
|
| 651 |
+
tip = "π― **Well calibrated** β confident and correct. That's the target behavior."
|
| 652 |
elif not rb.was_correct and confidence < 40:
|
| 653 |
+
tip = "π― **Good self-awareness** β you sensed your uncertainty correctly."
|
| 654 |
+
elif rb.underconfidence_penalty_val < -0.1:
|
| 655 |
+
tip = "π€ **Underconfident** β you got it right but doubted yourself. Trust your knowledge more."
|
| 656 |
else:
|
| 657 |
tip = ""
|
| 658 |
+
|
| 659 |
return result_md, stats_md, tip
|
| 660 |
|
| 661 |
|
| 662 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 663 |
+
# Tab 2
|
| 664 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 665 |
|
| 666 |
def run_comparison(scenario: str) -> tuple:
|
|
|
|
|
|
|
|
|
|
| 667 |
_init()
|
| 668 |
from core.baseline import AlwaysHighAgent, HeuristicAgent
|
| 669 |
from env.reward import compute_reward, RewardHistory
|
| 670 |
from env.parser import format_prompt, parse_response
|
| 671 |
from core.metrics import compute_report
|
| 672 |
|
| 673 |
+
domain_map = {
|
| 674 |
+
"Math": "math", "Logic": "logic", "Factual": "factual",
|
| 675 |
+
"Science": "science", "Medical": "medical", "Coding": "coding",
|
| 676 |
+
"Creative": "creative", "Mixed": None,
|
| 677 |
+
}
|
| 678 |
domain = domain_map.get(scenario)
|
| 679 |
n = 10
|
| 680 |
|
| 681 |
+
baseline = AlwaysHighAgent()
|
| 682 |
echo_agent = HeuristicAgent()
|
|
|
|
| 683 |
echo_h, base_h = RewardHistory(), RewardHistory()
|
| 684 |
+
rows_html = ""
|
| 685 |
+
|
| 686 |
for i in range(n):
|
| 687 |
+
d = domain or cfg.DOMAINS[i % len(cfg.DOMAINS)]
|
| 688 |
task = _task_bank.get_task(d, "medium")
|
| 689 |
prompt = format_prompt(task["question"], d, "medium")
|
| 690 |
|
| 691 |
ea = echo_agent(prompt); ep = parse_response(ea)
|
| 692 |
ba = baseline(prompt); bp = parse_response(ba)
|
| 693 |
+
er = compute_reward(ep.confidence, ep.answer, task["answer"],
|
| 694 |
+
task.get("answer_aliases", []), d)
|
| 695 |
+
br = compute_reward(bp.confidence, bp.answer, task["answer"],
|
| 696 |
+
task.get("answer_aliases", []), d)
|
| 697 |
|
| 698 |
echo_h.append(ep.confidence, er.was_correct, d, "medium", er.total)
|
| 699 |
base_h.append(bp.confidence, br.was_correct, d, "medium", br.total)
|
| 700 |
|
| 701 |
ei = "β
" if er.was_correct else "β"
|
| 702 |
bi = "β
" if br.was_correct else "β"
|
| 703 |
+
ec = "#00ffa3" if er.was_correct else "#ff4466"
|
| 704 |
+
bc = "#ff4466" if not br.was_correct else "#00ffa3"
|
| 705 |
+
|
| 706 |
+
rows_html += (
|
| 707 |
+
f"<div style='display:grid;grid-template-columns:1fr 1fr;gap:8px;margin-bottom:8px;'>"
|
| 708 |
+
f"<div style='background:rgba(0,255,163,0.04);border:1px solid rgba(0,255,163,0.12);"
|
| 709 |
+
f"border-radius:8px;padding:10px 14px;'>"
|
| 710 |
+
f"<div style='font-size:11px;color:#3d5a44;text-transform:uppercase;"
|
| 711 |
+
f"letter-spacing:0.08em;margin-bottom:4px;'>ECHO β {d} Q{i+1}</div>"
|
| 712 |
+
f"<div style='color:#8899bb;font-size:12px;margin-bottom:6px;'>"
|
| 713 |
+
f"{task['question'][:65]}β¦</div>"
|
| 714 |
+
f"<div style='display:flex;gap:8px;align-items:center;'>"
|
| 715 |
+
f"<span style='color:{ec};font-weight:700;font-size:15px;'>{ei}</span>"
|
| 716 |
+
f"<span style='background:rgba(0,255,163,0.1);border-radius:4px;"
|
| 717 |
+
f"padding:2px 8px;color:#00ffa3;font-size:12px;font-weight:600;'>"
|
| 718 |
+
f"conf: {ep.confidence}%</span></div></div>"
|
| 719 |
+
f"<div style='background:rgba(255,68,102,0.04);border:1px solid rgba(255,68,102,0.12);"
|
| 720 |
+
f"border-radius:8px;padding:10px 14px;'>"
|
| 721 |
+
f"<div style='font-size:11px;color:#5a2030;text-transform:uppercase;"
|
| 722 |
+
f"letter-spacing:0.08em;margin-bottom:4px;'>OVERCONFIDENT AI β Q{i+1}</div>"
|
| 723 |
+
f"<div style='color:#8899bb;font-size:12px;margin-bottom:6px;'>"
|
| 724 |
+
f"{task['question'][:65]}β¦</div>"
|
| 725 |
+
f"<div style='display:flex;gap:8px;align-items:center;'>"
|
| 726 |
+
f"<span style='color:{bc};font-weight:700;font-size:15px;'>{bi}</span>"
|
| 727 |
+
f"<span style='background:rgba(255,68,102,0.1);border-radius:4px;"
|
| 728 |
+
f"padding:2px 8px;color:#ff4466;font-size:12px;font-weight:600;'>"
|
| 729 |
+
f"conf: {bp.confidence}%</span></div></div>"
|
| 730 |
+
f"</div>"
|
| 731 |
+
)
|
| 732 |
|
| 733 |
+
em = echo_h.get_training_snapshot()
|
| 734 |
+
bm = base_h.get_training_snapshot()
|
| 735 |
+
delta_ece = abs(em['ece'] - bm['ece'])
|
| 736 |
+
|
| 737 |
+
summary_html = (
|
| 738 |
+
f"<div style='background:rgba(255,255,255,0.02);border:1px solid rgba(80,100,255,0.15);"
|
| 739 |
+
f"border-radius:10px;padding:20px;margin-top:4px;'>"
|
| 740 |
+
f"<div style='font-size:12px;font-weight:700;color:#4a5a8a;"
|
| 741 |
+
f"text-transform:uppercase;letter-spacing:0.08em;margin-bottom:16px;'>Results Summary</div>"
|
| 742 |
+
f"<div style='display:grid;grid-template-columns:repeat(4,1fr);gap:10px;margin-bottom:16px;'>"
|
| 743 |
+
+ _metric_card("ECE", f"{em['ece']:.3f}", f"{bm['ece']:.3f}", "#00ffa3", "#ff4466", "lower = better")
|
| 744 |
+
+ _metric_card("Accuracy", f"{em['accuracy']:.1%}", f"{bm['accuracy']:.1%}", "#00ffa3", "#ff4466", "")
|
| 745 |
+
+ _metric_card("Mean Conf", f"{em['mean_confidence']:.0f}%", f"{bm['mean_confidence']:.0f}%", "#4488ff", "#ff8c00", "")
|
| 746 |
+
+ _metric_card("Overconf Rate", f"{em['overconfidence_rate']:.1%}", f"{bm['overconfidence_rate']:.1%}", "#00ffa3", "#ff4466", "")
|
| 747 |
+
+ f"</div>"
|
| 748 |
+
f"<div style='background:linear-gradient(135deg,rgba(0,255,163,0.08),rgba(68,136,255,0.05));"
|
| 749 |
+
f"border:1px solid rgba(0,255,163,0.2);border-radius:8px;padding:12px 16px;text-align:center;'>"
|
| 750 |
+
f"<span style='color:#00ffa3;font-size:18px;font-weight:800;'>"
|
| 751 |
+
f"ECHO is {delta_ece:.0%} better calibrated</span>"
|
| 752 |
+
f"<span style='color:#4a5a8a;font-size:13px;'> than the overconfident baseline</span>"
|
| 753 |
+
f"</div></div>"
|
| 754 |
)
|
| 755 |
|
| 756 |
# Mini reliability diagram
|
| 757 |
+
erep = echo_h.get_calibration_report()
|
| 758 |
+
brep = base_h.get_calibration_report()
|
| 759 |
+
fig, ax = plt.subplots(figsize=(7, 4.5), facecolor="#04040e")
|
| 760 |
+
ax.set_facecolor("#080820")
|
| 761 |
+
ax.plot([0,100],[0,100],"--",color="#334455",alpha=0.6,linewidth=1.5,label="Perfect calibration",zorder=1)
|
| 762 |
+
for rep, col, lbl in [(erep,"#00ffa3","ECHO"),(brep,"#ff4466","Overconfident AI")]:
|
| 763 |
bd = rep.bin_data
|
| 764 |
+
xs = sorted(bd.keys())
|
| 765 |
+
ys = [bd[b]["accuracy"]*100 for b in xs]
|
| 766 |
+
if xs:
|
| 767 |
+
ax.plot(xs, ys, "-o", color=col, linewidth=2.5, markersize=7,
|
| 768 |
+
label=f"{lbl} ECE={rep.ece:.2f}", zorder=3,
|
| 769 |
+
markerfacecolor=col, markeredgecolor="#04040e", markeredgewidth=1.5)
|
| 770 |
+
ax.set_xlabel("Stated Confidence (%)", color="#4a5a8a", fontsize=11)
|
| 771 |
+
ax.set_ylabel("Actual Accuracy (%)", color="#4a5a8a", fontsize=11)
|
| 772 |
+
ax.set_title("Live Reliability Diagram", color="#c0d0ff", fontsize=13, fontweight="bold")
|
| 773 |
+
ax.tick_params(colors="#3a4a6a"); ax.set_xlim(0,100); ax.set_ylim(0,100)
|
| 774 |
+
for spine in ax.spines.values(): spine.set_color("#1a1a3a")
|
| 775 |
+
ax.grid(True, linestyle="--", alpha=0.12, color="#2a2a4a")
|
| 776 |
+
ax.legend(facecolor="#080820", labelcolor="#8899bb", edgecolor="#1a1a3a", fontsize=10)
|
| 777 |
+
plt.tight_layout()
|
| 778 |
tmp = tempfile.NamedTemporaryFile(suffix=".png", delete=False)
|
| 779 |
+
plt.savefig(tmp.name, dpi=120, bbox_inches="tight", facecolor="#04040e")
|
| 780 |
plt.close(fig)
|
| 781 |
|
| 782 |
+
return "<div style='display:flex;flex-direction:column;gap:4px;'>" + rows_html + "</div>" + summary_html, tmp.name
|
| 783 |
+
|
| 784 |
+
|
| 785 |
+
def _metric_card(label, echo_val, base_val, echo_col, base_col, note):
|
| 786 |
+
return (
|
| 787 |
+
f"<div style='background:rgba(255,255,255,0.02);border:1px solid rgba(80,100,255,0.1);"
|
| 788 |
+
f"border-radius:8px;padding:12px;text-align:center;'>"
|
| 789 |
+
f"<div style='font-size:11px;color:#3a4a6a;text-transform:uppercase;"
|
| 790 |
+
f"letter-spacing:0.07em;margin-bottom:6px;'>{label}</div>"
|
| 791 |
+
f"<div style='display:flex;justify-content:center;gap:12px;align-items:baseline;'>"
|
| 792 |
+
f"<span style='color:{echo_col};font-size:16px;font-weight:800;'>{echo_val}</span>"
|
| 793 |
+
f"<span style='color:#2a3a5a;font-size:12px;'>vs</span>"
|
| 794 |
+
f"<span style='color:{base_col};font-size:16px;font-weight:800;'>{base_val}</span>"
|
| 795 |
+
f"</div>"
|
| 796 |
+
f"{'<div style=color:#2a3a5a;font-size:10px;margin-top:3px;>'+note+'</div>' if note else ''}"
|
| 797 |
+
f"</div>"
|
| 798 |
+
)
|
| 799 |
|
| 800 |
|
| 801 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 802 |
+
# Tab 3
|
| 803 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 804 |
|
| 805 |
def generate_fingerprint(model_label: str) -> tuple:
|
| 806 |
from core.epistemic_fingerprint import _make_synthetic_fingerprint, plot_radar
|
| 807 |
_init()
|
| 808 |
offset_map = {"Untrained": 0.30, "ECHO Trained": 0.0, "Heuristic": 0.15}
|
| 809 |
+
fp = _make_synthetic_fingerprint(offset_map.get(model_label, 0.15), model_label)
|
| 810 |
baseline_fp = _make_synthetic_fingerprint(0.30, "Untrained")
|
| 811 |
|
| 812 |
tmp = tempfile.NamedTemporaryFile(suffix=".png", delete=False)
|
|
|
|
| 814 |
|
| 815 |
strongest = fp.strongest_domain.capitalize()
|
| 816 |
weakest = fp.weakest_domain.capitalize()
|
| 817 |
+
|
| 818 |
+
rows_html = (
|
| 819 |
+
"<div style='display:flex;flex-direction:column;gap:6px;'>"
|
| 820 |
+
)
|
| 821 |
for d in cfg.DOMAINS:
|
| 822 |
score = fp.domain_scores.get(d, 0.5)
|
| 823 |
ece_v = 1 - score
|
| 824 |
+
col = "#00ffa3" if score > 0.75 else ("#ffbb00" if score > 0.55 else "#ff4466")
|
| 825 |
+
pct = int(score * 100)
|
| 826 |
+
rows_html += (
|
| 827 |
+
f"<div style='display:flex;align-items:center;gap:10px;'>"
|
| 828 |
+
f"<div style='width:80px;color:#6677aa;font-size:13px;font-weight:500;"
|
| 829 |
+
f"text-align:right;'>{d.capitalize()}</div>"
|
| 830 |
+
f"<div style='flex:1;background:rgba(255,255,255,0.05);border-radius:4px;height:8px;'>"
|
| 831 |
+
f"<div style='width:{pct}%;height:100%;border-radius:4px;"
|
| 832 |
+
f"background:{col};box-shadow:0 0 8px {col}55;'></div></div>"
|
| 833 |
+
f"<div style='width:40px;color:{col};font-size:12px;font-weight:700;"
|
| 834 |
+
f"text-align:right;'>{score:.2f}</div>"
|
| 835 |
+
f"<div style='width:40px;color:#3a4a6a;font-size:11px;"
|
| 836 |
+
f"text-align:right;'>ECE {ece_v:.2f}</div>"
|
| 837 |
+
f"</div>"
|
| 838 |
+
)
|
| 839 |
+
rows_html += "</div>"
|
| 840 |
+
|
| 841 |
+
insight_html = (
|
| 842 |
+
f"<div style='background:rgba(168,85,247,0.06);border:1px solid rgba(168,85,247,0.2);"
|
| 843 |
+
f"border-radius:8px;padding:14px 16px;margin-top:4px;'>"
|
| 844 |
+
f"<div style='font-size:13px;color:#c0d0ff;line-height:1.6;'>"
|
| 845 |
+
f"<strong style='color:#a855f7;'>{model_label}</strong> is strongest in "
|
| 846 |
+
f"<strong style='color:#00ffa3;'>{strongest}</strong> and most uncertain in "
|
| 847 |
+
f"<strong style='color:#ff4466;'>{weakest}</strong>.</div>"
|
| 848 |
+
f"<div style='margin-top:8px;font-size:14px;color:#6677aa;'>"
|
| 849 |
+
f"Overall ECE: <strong style='color:#ffd700;'>{fp.overall_ece:.3f}</strong></div></div>"
|
| 850 |
)
|
| 851 |
+
|
| 852 |
+
return tmp.name, rows_html, insight_html
|
| 853 |
|
| 854 |
|
| 855 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 856 |
+
# Tab 5
|
| 857 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 858 |
|
| 859 |
def run_evaluation() -> tuple:
|
| 860 |
_init()
|
| 861 |
+
from core.tasks import TASKS, TaskRunner, TASKS_BY_ID
|
| 862 |
from core.baseline import HeuristicAgent
|
| 863 |
runner = TaskRunner()
|
| 864 |
agent = HeuristicAgent()
|
| 865 |
result = runner.run_all(agent, _task_bank)
|
| 866 |
+
|
| 867 |
+
rows_html = ""
|
| 868 |
for r in result.tasks:
|
|
|
|
| 869 |
t = TASKS_BY_ID[r.task_id]
|
| 870 |
+
ok = r.passed
|
| 871 |
+
col = "#00ffa3" if ok else "#ff4466"
|
| 872 |
+
bg = "rgba(0,255,163,0.05)" if ok else "rgba(255,68,102,0.05)"
|
| 873 |
+
border = "rgba(0,255,163,0.2)" if ok else "rgba(255,68,102,0.2)"
|
| 874 |
+
icon = "β
PASS" if ok else "β FAIL"
|
| 875 |
+
pct = min(int(r.score / t.pass_threshold * 100), 100)
|
| 876 |
+
rows_html += (
|
| 877 |
+
f"<div style='background:{bg};border:1px solid {border};"
|
| 878 |
+
f"border-radius:10px;padding:16px 20px;margin-bottom:8px;'>"
|
| 879 |
+
f"<div style='display:flex;justify-content:space-between;align-items:center;"
|
| 880 |
+
f"margin-bottom:10px;'>"
|
| 881 |
+
f"<div>"
|
| 882 |
+
f"<span style='color:{col};font-weight:700;font-size:15px;'>{icon}</span>"
|
| 883 |
+
f"<span style='color:#c0d0ff;font-size:14px;font-weight:600;margin-left:10px;'>"
|
| 884 |
+
f"{t.name}</span>"
|
| 885 |
+
f"</div>"
|
| 886 |
+
f"<div style='font-family:JetBrains Mono,monospace;font-size:13px;'>"
|
| 887 |
+
f"<span style='color:{col};font-weight:700;'>{r.score:.3f}</span>"
|
| 888 |
+
f"<span style='color:#2a3a5a;'> / {t.pass_threshold}</span>"
|
| 889 |
+
f"</div></div>"
|
| 890 |
+
f"<div style='background:rgba(255,255,255,0.04);border-radius:4px;height:6px;'>"
|
| 891 |
+
f"<div style='width:{pct}%;height:100%;border-radius:4px;"
|
| 892 |
+
f"background:{col};'></div></div>"
|
| 893 |
+
f"</div>"
|
| 894 |
+
)
|
| 895 |
+
|
| 896 |
+
verdict_color = "#00ffa3" if result.overall_pass else "#ff4466"
|
| 897 |
+
verdict_html = (
|
| 898 |
+
f"<div style='background:linear-gradient(135deg,rgba(0,255,163,0.08),rgba(68,136,255,0.05));"
|
| 899 |
+
f"border:1px solid {verdict_color}44;border-radius:10px;padding:16px 20px;"
|
| 900 |
+
f"text-align:center;margin-top:4px;'>"
|
| 901 |
+
f"<div style='font-size:20px;font-weight:800;color:{verdict_color};'>"
|
| 902 |
+
f"{'π ALL TASKS PASSED' if result.overall_pass else 'β οΈ Some tasks need improvement'}"
|
| 903 |
+
f"</div></div>"
|
| 904 |
+
)
|
| 905 |
+
|
| 906 |
json_str = json.dumps(result.to_dict(), indent=2, default=str)
|
| 907 |
+
return rows_html + verdict_html, json_str
|
| 908 |
|
| 909 |
|
| 910 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 911 |
+
# Build app
|
| 912 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 913 |
|
| 914 |
def build_app():
|
| 915 |
import gradio as gr
|
| 916 |
|
| 917 |
plots = {k: f"{cfg.PLOTS_DIR}/{v}" for k, v in {
|
| 918 |
+
"reliability": "reliability_diagram.png",
|
| 919 |
+
"training": "training_curves.png",
|
| 920 |
+
"fingerprint": "epistemic_fingerprint.png",
|
| 921 |
+
"heatmap": "calibration_heatmap.png",
|
| 922 |
+
"distribution": "confidence_distribution.png",
|
| 923 |
+
"domain": "domain_comparison.png",
|
| 924 |
}.items()}
|
|
|
|
| 925 |
def _img(key): return plots[key] if Path(plots[key]).exists() else None
|
| 926 |
|
| 927 |
+
with gr.Blocks(title="ECHO ULTIMATE") as demo:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 928 |
|
| 929 |
+
# ββ Hero βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 930 |
+
gr.HTML(_HERO_HTML)
|
| 931 |
+
|
| 932 |
+
# ββ Tab 1: Live Challenge βββββββββββββββββββββββββββββββββββββββββββββ
|
| 933 |
+
with gr.Tab("π― Live Challenge"):
|
| 934 |
+
gr.HTML(_section_header(
|
| 935 |
+
"π― Live Challenge",
|
| 936 |
+
"Answer questions with a confidence score β discover how well-calibrated you are",
|
| 937 |
+
"#00ffa3"
|
| 938 |
+
))
|
| 939 |
with gr.Row():
|
| 940 |
+
dom_dd = gr.Dropdown(
|
| 941 |
+
["Math","Logic","Factual","Science","Medical","Coding","Creative"],
|
| 942 |
+
value="Math", label="Domain"
|
| 943 |
+
)
|
| 944 |
diff_dd = gr.Dropdown(["Easy","Medium","Hard"], value="Easy", label="Difficulty")
|
| 945 |
+
get_btn = gr.Button("π² Get Question", variant="primary", scale=1)
|
| 946 |
+
|
| 947 |
+
question_box = gr.Markdown(
|
| 948 |
+
"<div style='color:#3a4a6a;font-style:italic;padding:12px;'>"
|
| 949 |
+
"Select a domain and difficulty, then click Get Question.</div>"
|
| 950 |
+
)
|
| 951 |
+
|
| 952 |
with gr.Row():
|
| 953 |
+
with gr.Column(scale=2):
|
| 954 |
+
conf_sl = gr.Slider(0, 100, value=50, step=5,
|
| 955 |
+
label="Confidence (0 = no idea Β· 100 = certain)")
|
| 956 |
+
with gr.Column(scale=3):
|
| 957 |
+
ans_box = gr.Textbox(label="Your Answer", placeholder="Type your answerβ¦",
|
| 958 |
+
lines=1)
|
| 959 |
+
|
| 960 |
+
sub_btn = gr.Button("β
Submit Answer", variant="primary")
|
| 961 |
+
|
| 962 |
with gr.Row():
|
| 963 |
+
result_html = gr.HTML()
|
| 964 |
+
stats_html = gr.HTML()
|
| 965 |
tip_md = gr.Markdown()
|
| 966 |
+
|
| 967 |
get_btn.click(get_question, [dom_dd, diff_dd], [question_box, ans_box])
|
| 968 |
+
sub_btn.click(submit_answer, [conf_sl, ans_box], [result_html, stats_html, tip_md])
|
| 969 |
+
|
| 970 |
+
# ββ Tab 2: Battle βββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 971 |
+
with gr.Tab("β ECHO vs Overconfident AI"):
|
| 972 |
+
gr.HTML(_section_header(
|
| 973 |
+
"β ECHO vs Overconfident AI",
|
| 974 |
+
"10-question head-to-head: calibrated ECHO vs AlwaysHigh baseline (always 90% confident)",
|
| 975 |
+
"#ff4466"
|
| 976 |
+
))
|
| 977 |
+
with gr.Row():
|
| 978 |
+
scenario_dd = gr.Dropdown(
|
| 979 |
+
["Mixed","Math","Logic","Factual","Science","Medical","Coding","Creative"],
|
| 980 |
+
value="Mixed", label="Test Scenario"
|
| 981 |
+
)
|
| 982 |
+
run_btn = gr.Button("β Run 10 Questions", variant="primary")
|
| 983 |
+
|
| 984 |
+
with gr.Row():
|
| 985 |
+
with gr.Column(scale=3):
|
| 986 |
+
cmp_html = gr.HTML()
|
| 987 |
+
with gr.Column(scale=2):
|
| 988 |
+
mini_img = gr.Image(label="Live Reliability Diagram", type="filepath",
|
| 989 |
+
show_label=True, height=320)
|
| 990 |
+
|
| 991 |
+
run_btn.click(run_comparison, [scenario_dd], [cmp_html, mini_img])
|
| 992 |
+
|
| 993 |
+
# ββ Tab 3: Fingerprint βββββββββββββββββββββοΏ½οΏ½ββββββββββββββββββββββββββ
|
| 994 |
+
with gr.Tab("𧬠Epistemic Fingerprint"):
|
| 995 |
+
gr.HTML(_section_header(
|
| 996 |
+
"𧬠Epistemic Fingerprint",
|
| 997 |
+
"Radar chart of calibration across all 7 domains β larger green = better everywhere",
|
| 998 |
+
"#a855f7"
|
| 999 |
+
))
|
| 1000 |
+
with gr.Row():
|
| 1001 |
+
model_dd = gr.Dropdown(
|
| 1002 |
+
["ECHO Trained","Untrained","Heuristic"],
|
| 1003 |
+
value="ECHO Trained", label="Model"
|
| 1004 |
+
)
|
| 1005 |
+
fp_btn = gr.Button("π¬ Generate Fingerprint", variant="primary")
|
| 1006 |
+
|
| 1007 |
+
with gr.Row():
|
| 1008 |
+
with gr.Column(scale=3):
|
| 1009 |
+
fp_img = gr.Image(label="Epistemic Fingerprint", type="filepath",
|
| 1010 |
+
value=_img("fingerprint"), height=480)
|
| 1011 |
+
with gr.Column(scale=2):
|
| 1012 |
+
fp_bars = gr.HTML()
|
| 1013 |
+
fp_insight = gr.HTML()
|
| 1014 |
+
|
| 1015 |
+
fp_btn.click(generate_fingerprint, [model_dd], [fp_img, fp_bars, fp_insight])
|
| 1016 |
+
|
| 1017 |
+
# ββ Tab 4: Training Evidence ββββββββββββββββββββββββββββββββββββββββββ
|
| 1018 |
+
with gr.Tab("π Training Evidence"):
|
| 1019 |
+
gr.HTML(_section_header(
|
| 1020 |
+
"π Training Evidence",
|
| 1021 |
+
"6 plots generated from GRPO training β from random overconfidence to precise calibration",
|
| 1022 |
+
"#ffd700"
|
| 1023 |
+
))
|
| 1024 |
+
|
| 1025 |
+
gr.HTML("""
|
| 1026 |
+
<div style='background:rgba(0,255,163,0.05);border:1px solid rgba(0,255,163,0.2);
|
| 1027 |
+
border-radius:10px;padding:16px 20px;margin-bottom:8px;'>
|
| 1028 |
+
<div style='font-size:15px;font-weight:700;color:#00ffa3;margin-bottom:6px;'>
|
| 1029 |
+
β
Hero Plot β Reliability Diagram</div>
|
| 1030 |
+
<div style='color:#6677aa;font-size:13px;'>
|
| 1031 |
+
The smoking gun. Untrained model (red): flat line far from the diagonal β always overconfident.
|
| 1032 |
+
ECHO trained (green): hugs the perfect calibration diagonal.
|
| 1033 |
+
</div>
|
| 1034 |
+
</div>""")
|
| 1035 |
+
gr.Image(value=_img("reliability"), label="Reliability Diagram", height=380)
|
| 1036 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1037 |
with gr.Row():
|
| 1038 |
with gr.Column():
|
| 1039 |
+
gr.HTML("<div style='font-size:13px;font-weight:600;color:#4488ff;"
|
| 1040 |
+
"margin:8px 0 4px;'>π Training Curves</div>"
|
| 1041 |
+
"<div style='font-size:12px;color:#4a5a8a;margin-bottom:8px;'>"
|
| 1042 |
+
"ECE drops 0.34 β 0.08 across 3 curriculum phases</div>")
|
| 1043 |
+
gr.Image(value=_img("training"), label="Training Curves", height=300)
|
| 1044 |
with gr.Column():
|
| 1045 |
+
gr.HTML("<div style='font-size:13px;font-weight:600;color:#a855f7;"
|
| 1046 |
+
"margin:8px 0 4px;'>𧬠Epistemic Fingerprint</div>"
|
| 1047 |
+
"<div style='font-size:12px;color:#4a5a8a;margin-bottom:8px;'>"
|
| 1048 |
+
"Domain-level calibration β green fills every axis</div>")
|
| 1049 |
+
gr.Image(value=_img("fingerprint"), label="Epistemic Fingerprint", height=300)
|
| 1050 |
+
|
| 1051 |
with gr.Row():
|
| 1052 |
with gr.Column():
|
| 1053 |
+
gr.HTML("<div style='font-size:13px;font-weight:600;color:#ffd700;"
|
| 1054 |
+
"margin:8px 0 4px;'>π‘οΈ Calibration Heatmap</div>"
|
| 1055 |
+
"<div style='font-size:12px;color:#4a5a8a;margin-bottom:8px;'>"
|
| 1056 |
+
"7 domains Γ 3 difficulties β red=bad, green=good</div>")
|
| 1057 |
+
gr.Image(value=_img("heatmap"), label="Calibration Heatmap", height=300)
|
| 1058 |
with gr.Column():
|
| 1059 |
+
gr.HTML("<div style='font-size:13px;font-weight:600;color:#ff8c00;"
|
| 1060 |
+
"margin:8px 0 4px;'>π Confidence Distribution</div>"
|
| 1061 |
+
"<div style='font-size:12px;color:#4a5a8a;margin-bottom:8px;'>"
|
| 1062 |
+
"Untrained: spike at 85β95%. ECHO: spread = actual accuracy</div>")
|
| 1063 |
+
gr.Image(value=_img("distribution"), label="Confidence Distribution", height=300)
|
| 1064 |
+
|
| 1065 |
+
gr.HTML("<div style='font-size:13px;font-weight:600;color:#ff4466;"
|
| 1066 |
+
"margin:8px 0 4px;'>π’ Domain Comparison</div>"
|
| 1067 |
+
"<div style='font-size:12px;color:#4a5a8a;margin-bottom:8px;'>"
|
| 1068 |
+
"ECE improvement across all 7 domains</div>")
|
| 1069 |
+
gr.Image(value=_img("domain"), label="Domain Comparison", height=320)
|
| 1070 |
+
|
| 1071 |
+
regen_btn = gr.Button("π Regenerate All Plots", variant="secondary")
|
| 1072 |
+
regen_status = gr.HTML()
|
| 1073 |
|
| 1074 |
def regen():
|
| 1075 |
from training.evaluate import make_synthetic_pair, compare_and_plot
|
| 1076 |
before, after = make_synthetic_pair()
|
| 1077 |
paths = compare_and_plot(after, {"Untrained": before})
|
| 1078 |
+
html = ("<div style='color:#00ffa3;font-size:13px;font-weight:600;"
|
| 1079 |
+
"padding:8px 12px;background:rgba(0,255,163,0.06);"
|
| 1080 |
+
"border-radius:6px;'>β
All 6 plots regenerated</div>")
|
| 1081 |
+
return html
|
| 1082 |
+
|
| 1083 |
+
regen_btn.click(regen, outputs=[regen_status])
|
| 1084 |
+
|
| 1085 |
+
# ββ Tab 5: Evaluation βββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1086 |
+
with gr.Tab("π Official Evaluation"):
|
| 1087 |
+
gr.HTML(_section_header(
|
| 1088 |
+
"π Official OpenEnv Evaluation",
|
| 1089 |
+
"3 tasks Γ 30 episodes β validates ECHO meets the benchmark thresholds",
|
| 1090 |
+
"#ffd700"
|
| 1091 |
+
))
|
| 1092 |
+
gr.HTML("""
|
| 1093 |
+
<div style='display:grid;grid-template-columns:repeat(3,1fr);gap:10px;margin-bottom:8px;'>
|
| 1094 |
+
<div style='background:rgba(68,136,255,0.06);border:1px solid rgba(68,136,255,0.15);
|
| 1095 |
+
border-radius:8px;padding:12px 16px;'>
|
| 1096 |
+
<div style='color:#4488ff;font-weight:700;font-size:13px;'>Task 1 β Easy</div>
|
| 1097 |
+
<div style='color:#3a4a6a;font-size:12px;margin-top:4px;'>ECE target: < 0.15</div>
|
| 1098 |
+
</div>
|
| 1099 |
+
<div style='background:rgba(255,187,0,0.06);border:1px solid rgba(255,187,0,0.15);
|
| 1100 |
+
border-radius:8px;padding:12px 16px;'>
|
| 1101 |
+
<div style='color:#ffbb00;font-weight:700;font-size:13px;'>Task 2 β Medium</div>
|
| 1102 |
+
<div style='color:#3a4a6a;font-size:12px;margin-top:4px;'>ECE target: < 0.20</div>
|
| 1103 |
+
</div>
|
| 1104 |
+
<div style='background:rgba(168,85,247,0.06);border:1px solid rgba(168,85,247,0.15);
|
| 1105 |
+
border-radius:8px;padding:12px 16px;'>
|
| 1106 |
+
<div style='color:#a855f7;font-weight:700;font-size:13px;'>Task 3 β Hard</div>
|
| 1107 |
+
<div style='color:#3a4a6a;font-size:12px;margin-top:4px;'>ECE target: < 0.25</div>
|
| 1108 |
+
</div>
|
| 1109 |
+
</div>""")
|
| 1110 |
+
eval_btn = gr.Button("π Run Full Evaluation (90 episodes)", variant="primary")
|
| 1111 |
+
result_html = gr.HTML()
|
| 1112 |
+
with gr.Accordion("π Raw JSON output", open=False):
|
| 1113 |
json_out = gr.Code(language="json")
|
| 1114 |
+
eval_btn.click(run_evaluation, outputs=[result_html, json_out])
|
| 1115 |
+
|
| 1116 |
+
# ββ Tab 6: Live Training βββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1117 |
+
with gr.Tab("β‘ Live Training"):
|
| 1118 |
+
gr.HTML(_section_header(
|
| 1119 |
+
"β‘ Live GRPO Training",
|
| 1120 |
+
"Watch ECE drop in real-time as the model trains. Dashed lines = pass thresholds.",
|
| 1121 |
+
"#4488ff"
|
| 1122 |
+
))
|
|
|
|
| 1123 |
with gr.Row():
|
| 1124 |
+
lt_start_btn = gr.Button("π Start Live Training Demo", variant="primary", scale=2)
|
| 1125 |
+
lt_stop_btn = gr.Button("βΉ Stop", variant="stop", scale=1)
|
| 1126 |
+
|
| 1127 |
+
lt_status = gr.Textbox(
|
| 1128 |
+
label="Training Log",
|
| 1129 |
+
value="Ready β click Start to simulate GRPO training.",
|
| 1130 |
+
lines=2, interactive=False,
|
| 1131 |
+
elem_classes=["terminal-box"],
|
| 1132 |
+
)
|
| 1133 |
+
lt_plot = gr.Image(
|
| 1134 |
+
label="ECE During Training",
|
| 1135 |
+
type="filepath", height=380,
|
| 1136 |
)
|
|
|
|
|
|
|
| 1137 |
lt_progress = gr.Slider(
|
| 1138 |
minimum=0, maximum=100, value=0,
|
| 1139 |
+
label="Progress (%)", interactive=False,
|
| 1140 |
)
|
| 1141 |
|
| 1142 |
+
lt_start_btn.click(start_live_training,
|
| 1143 |
+
outputs=[lt_status, lt_plot, lt_progress])
|
|
|
|
|
|
|
| 1144 |
lt_stop_btn.click(stop_live_training, outputs=[lt_status])
|
| 1145 |
|
| 1146 |
return demo
|
| 1147 |
|
| 1148 |
|
| 1149 |
def main():
|
| 1150 |
+
import gradio as gr
|
| 1151 |
logging.basicConfig(level=logging.INFO)
|
| 1152 |
demo = build_app()
|
| 1153 |
+
demo.launch(
|
| 1154 |
+
server_name="0.0.0.0",
|
| 1155 |
+
server_port=cfg.GRADIO_PORT,
|
| 1156 |
+
share=False,
|
| 1157 |
+
show_error=True,
|
| 1158 |
+
css=_CSS,
|
| 1159 |
+
theme=gr.themes.Base(
|
| 1160 |
+
primary_hue=gr.themes.colors.blue,
|
| 1161 |
+
neutral_hue=gr.themes.colors.slate,
|
| 1162 |
+
font=[gr.themes.GoogleFont("Inter"), "sans-serif"],
|
| 1163 |
+
),
|
| 1164 |
+
)
|
| 1165 |
|
| 1166 |
|
| 1167 |
if __name__ == "__main__":
|