Added scaffolding for evals

CLAUDE.md

@@ -103,6 +103,9 @@ Copy `.env.example` → `.env` and set:
 
 ## Development Notes
 
+- **NEVER use local Ollama models** (e.g. `qwen3:8b`, `gemma3:1b`) — this machine
+  is not powerful enough and will break. Always use cloud-backed models like
+  `qwen3.5:397b-cloud` or `gpt-oss:20b-cloud` via Ollama, or vLLM tiers.
 - **Adding a persona**: add to `PERSONAS` in `data/generate_users.py`, re-run it,
   then `python -m backend.retrieval.vector_store` to rebuild indexes
 - **Changing LLM**: set `ACTIVE_LLM_TIER` in `.env` — no code changes needed

README.md

@@ -184,11 +184,32 @@ To add a new persona, edit `data/generate_users.py` and re-run `python -m backen
 
 ## TODO
 
-- [ ] Add evals for performance
 - [ ] Add more dataset
 - [ ] Reduce latency in intention
 - [ ] Add more detailed todos
 
+### Evals (`backend/evals/`)
+
+Per-turn metrics returned in `ChatResponse.eval_scores` and rendered in the React debug panel.
+
+| Metric | File | Status |
+|--------|------|--------|
+| Communication Efficiency | `efficiency.py` | Done — SLO check on `t_total` |
+| Factual Faithfulness | `faithfulness.py` | Stub |
+| Multimodal Alignment | `multimodal_alignment.py` | Stub |
+| Perceived Authenticity | (frontend) | UI star rating; not persisted yet |
+
+- [ ] **Faithfulness** — Load cross-encoder NLI model (e.g. `cross-encoder/nli-deberta-v3-small`),
+  split response into sentences, check entailment against evidence chunks. Groundedness =
+  fraction with max entailment > 0.5; hallucination rate = fraction with contradiction > 0.5
+  and entailment < 0.3. Empty `chunks` → `no_evidence=True`.
+- [ ] **Multimodal Alignment** — Rule-based (no model):
+  - Affect → sentiment-word overlap (reuse `affect_positive_map` from planner)
+  - Gesture → expected-word overlap (reuse `gesture_word_map` from planner)
+  - Gaze → check whether retrieved chunks came from `gaze_bucket` and response references them
+  - Overall = mean of non-None sub-scores
+- [ ] **Authenticity** — Persist Likert ratings (currently client-side only). Add `POST /chat/rate`.
+
 ---
 
 ## Team

backend/api/main.py

@@ -8,7 +8,11 @@ from fastapi.middleware.cors import CORSMiddleware
 from pydantic import BaseModel
 
 from backend.config.settings import settings
-from backend.generation.llm_client import get_client
+from backend.evals import compute_evals
+from backend.generation.llm_client import (  # active_model used by /debug/config
+    active_model,
+    get_client,
+)
 from backend.guardrails.checks import check_input
 from backend.pipeline.graph import aac_graph
 from backend.pipeline.state import PipelineState
@@ -63,15 +67,31 @@ class ChatRequest(BaseModel):
     air_written_text: str | None = None
 
 
+class EvalScoresResponse(BaseModel):
+    groundedness: float
+    hallucination_rate: float
+    no_evidence: bool
+    t_total_s: float
+    slo_target_s: float
+    slo_passed: bool
+    slo_margin_s: float
+    multimodal_alignment: float
+    affect_alignment: float
+    gesture_alignment: float
+    gaze_alignment: float
+
+
 class ChatResponse(BaseModel):
     user_id: str
     query: str
     response: str
     affect: str
     llm_tier: str
+    llm_model: str
     retrieval_mode: str
     latency: dict
     guardrail_passed: bool
+    eval_scores: EvalScoresResponse | None = None
 
 
 # ── Helpers ────────────────────────────────────────────────────────────────────
@@ -123,6 +143,7 @@ def _build_initial_state(req: ChatRequest, session: dict) -> PipelineState:
         candidates=[],
         selected_response=None,
         llm_tier_used="",
+        llm_model_used="",
         latency_log={
             "t_sensing": 0.0,
             "t_intent": 0.0,
@@ -143,6 +164,23 @@ def health():
     return {"status": "ok", "models_ready": _models_ready}
 
 
+@app.get("/debug/config")
+def debug_config():
+    """Return active model + key settings for the debug panel."""
+    return {
+        "active_llm_tier": settings.active_llm_tier,
+        "active_model": active_model(),
+        "thinking_mode": settings.thinking_mode,
+        "embed_model": settings.embed_model,
+        "rerank_model": settings.rerank_model,
+        "retrieval_top_k": settings.retrieval_top_k,
+        "retrieval_rerank_k": settings.retrieval_rerank_k,
+        "fallback_latency_threshold": settings.fallback_latency_threshold,
+        "slo_target_s": settings.slo_target_s,
+        "num_candidates": settings.num_candidates,
+    }
+
+
 @app.get("/users")
 def list_users():
     try:
@@ -170,6 +208,7 @@ def chat(req: ChatRequest):
             response=guard["fallback"],
             affect="NEUTRAL",
             llm_tier="none",
+            llm_model="none",
             retrieval_mode="none",
             latency={},
             guardrail_passed=False,
@@ -184,13 +223,27 @@ def chat(req: ChatRequest):
     session["session_history"] = result["session_history"]
     session["bucket_priors"] = result["bucket_priors"]
 
+    # Compute evaluation metrics
+    affect_emotion = (result.get("affect") or {}).get("emotion", "NEUTRAL")
+    eval_scores = compute_evals(
+        response=result["selected_response"] or "",
+        chunks=result.get("retrieved_chunks") or [],
+        latency_log=result.get("latency_log") or {},
+        affect=affect_emotion,
+        gesture_tag=req.gesture_tag,
+        gaze_bucket=req.gaze_bucket,
+        slo_target=settings.slo_target_s,
+    )
+
     return ChatResponse(
         user_id=req.user_id,
         query=req.query,
         response=result["selected_response"] or "",
-        affect=(result.get("affect") or {}).get("emotion", "NEUTRAL"),
+        affect=affect_emotion,
         llm_tier=result.get("llm_tier_used", "unknown"),
+        llm_model=result.get("llm_model_used", "unknown"),
         retrieval_mode=result.get("retrieval_mode_used", "unknown"),
         latency=result.get("latency_log") or {},
         guardrail_passed=result.get("guardrail_passed", True),
+        eval_scores=eval_scores,
     )

backend/config/settings.py

@@ -69,5 +69,8 @@ class Settings(BaseSettings):
     rank_beta: float = 0.3  # style similarity weight
     rank_gamma: float = 0.3  # affect-match weight
 
+    # ── Evaluation ────────────────────────────────────────────────────────────
+    slo_target_s: float = 6.0  # max acceptable response latency (seconds)
+
 
 settings = Settings()

backend/evals/__init__.py

@@ -0,0 +1,37 @@
+# Evaluation metrics — compute after pipeline returns, before API response.
+from __future__ import annotations
+
+from backend.evals.efficiency import compute_efficiency
+from backend.evals.faithfulness import compute_faithfulness
+from backend.evals.multimodal_alignment import compute_multimodal_alignment
+
+
+def compute_evals(
+    response: str,
+    chunks: list[dict],
+    latency_log: dict,
+    affect: str | None,
+    gesture_tag: str | None,
+    gaze_bucket: str | None,
+    slo_target: float = 6.0,
+) -> dict:
+    """Run all eval scorers and return a unified EvalScores dict."""
+    faith = compute_faithfulness(response, chunks)
+    eff = compute_efficiency(latency_log, slo_target)
+    align = compute_multimodal_alignment(
+        response, affect, gesture_tag, gaze_bucket, chunks
+    )
+
+    return {
+        "groundedness": faith["groundedness"],
+        "hallucination_rate": faith["hallucination_rate"],
+        "no_evidence": faith["no_evidence"],
+        "t_total_s": eff["t_total"],
+        "slo_target_s": eff["slo_target"],
+        "slo_passed": eff["slo_passed"],
+        "slo_margin_s": eff["margin_s"],
+        "multimodal_alignment": align["overall_score"],
+        "affect_alignment": align["affect_alignment"],
+        "gesture_alignment": align["gesture_alignment"],
+        "gaze_alignment": align["gaze_alignment"],
+    }

backend/evals/efficiency.py

@@ -0,0 +1,13 @@
+# Communication efficiency — SLO pass/fail on response latency.
+from __future__ import annotations
+
+
+def compute_efficiency(latency_log: dict, slo_target: float = 6.0) -> dict:
+    """Check if total response time meets the SLO target."""
+    t_total = latency_log.get("t_total", 0.0)
+    return {
+        "t_total": round(t_total, 3),
+        "slo_target": slo_target,
+        "slo_passed": t_total < slo_target,
+        "margin_s": round(slo_target - t_total, 3),
+    }

backend/evals/faithfulness.py

@@ -0,0 +1,12 @@
+# NLI-based faithfulness scoring.
+from __future__ import annotations
+
+
+def compute_faithfulness(response: str, chunks: list[dict]) -> dict:
+    """Compute groundedness and hallucination rate via NLI."""
+    no_evidence = len(chunks) == 0
+    return {
+        "groundedness": 0.0,
+        "hallucination_rate": 0.0,
+        "no_evidence": no_evidence,
+    }

backend/evals/multimodal_alignment.py

@@ -0,0 +1,18 @@
+# Multimodal alignment scoring.
+from __future__ import annotations
+
+
+def compute_multimodal_alignment(
+    response: str,
+    affect: str | None,
+    gesture_tag: str | None,
+    gaze_bucket: str | None,
+    chunks: list[dict],
+) -> dict:
+    """Score alignment between non-verbal inputs and generated text."""
+    return {
+        "overall_score": 0.0,
+        "affect_alignment": 0.0,
+        "gesture_alignment": 0.0,
+        "gaze_alignment": 0.0,
+    }

backend/generation/llm_client.py

@@ -97,11 +97,19 @@ def chat_complete(
         **kwargs,
     )
     raw = (resp.choices[0].message.content if resp.choices else "") or ""
+    print(
+        f"[llm_client] tier={resolved_tier} model={model} raw_len={len(raw)} raw={raw[:200]!r}"
+    )
 
     if settings.thinking_mode in ("off", "strip"):
         raw = _strip_think_tags(raw)
 
-    return raw.strip()
+    stripped = raw.strip()
+    if not stripped:
+        print(
+            f"[llm_client] WARNING: empty response after strip. finish_reason={resp.choices[0].finish_reason if resp.choices else 'none'}"
+        )
+    return stripped
 
 
 def warmup(tier: str | None = None) -> None:

backend/pipeline/nodes/planner.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 import time
 
 from backend.config.settings import settings
-from backend.generation.llm_client import chat_complete
+from backend.generation.llm_client import active_model, chat_complete
 from backend.guardrails.checks import check_output
 from backend.pipeline.state import PipelineState
 from backend.sensing.gesture import GESTURE_TO_TAG
@@ -94,11 +94,16 @@ def _run(state: PipelineState, tier: str) -> dict:
         4,
     )
 
+    # Mirror chat_complete's tier collapsing so the reported model matches what ran.
+    actual_tier = "local" if settings.active_llm_tier == "local" else tier
+    actual_model = active_model(actual_tier)
+
     return {
         "augmented_prompt": prompt,
         "candidates": candidates,
         "selected_response": selected,
-        "llm_tier_used": tier,
+        "llm_tier_used": actual_tier,
+        "llm_model_used": actual_model,
         "latency_log": latency_log,
         "guardrail_passed": guard["passed"],
     }

backend/pipeline/state.py

@@ -90,6 +90,7 @@ class PipelineState(TypedDict):
     candidates: list[str]  # 2-3 candidate responses
     selected_response: str | None
     llm_tier_used: str  # "primary" | "fallback" | "local"
+    llm_model_used: str  # actual model name (e.g. "gemma4:31b-cloud")
 
     # ── L5: Feedback / tracking ───────────────────────────────────────────────
     latency_log: LatencyLog | None

frontend/src/App.css

@@ -124,14 +124,15 @@ select:focus, input[type="text"]:focus {
   font-weight: 500;
 }
 
-/* ── Latency metrics ──────────────────────────────────────────────── */
+/* ── Shared metric primitives (latency, eval) ─────────────────────── */
 
-.latency-metrics h3 {
+.section-title {
   font-size: 12px;
   color: #888;
   text-transform: uppercase;
   letter-spacing: 0.5px;
   margin-bottom: 6px;
+  font-weight: 600;
 }
 
 .metric-row {
@@ -139,6 +140,13 @@ select:focus, input[type="text"]:focus {
   justify-content: space-between;
   font-size: 13px;
   padding: 2px 0;
+  color: #aaa;
+}
+
+.metric-row.sub {
+  padding-left: 12px;
+  font-size: 11px;
+  color: #777;
 }
 
 .metric-label {
@@ -150,6 +158,14 @@ select:focus, input[type="text"]:focus {
   font-family: monospace;
 }
 
+.metric-value.pass {
+  color: #4caf50;
+}
+
+.metric-value.fail {
+  color: #f44336;
+}
+
 .no-metrics {
   color: #555;
   font-size: 13px;
@@ -260,3 +276,109 @@ select:focus, input[type="text"]:focus {
   color: #e55;
   font-size: 13px;
 }
+
+/* ── Eval panel ──────────────────────────────────────────────────── */
+
+.eval-panel {
+  margin-top: 8px;
+  border-top: 1px solid #3a3d47;
+  padding-top: 6px;
+}
+
+.eval-toggle {
+  background: none;
+  border: none;
+  color: #888;
+  font-size: 12px;
+  cursor: pointer;
+  padding: 2px 0;
+  display: flex;
+  align-items: center;
+  gap: 8px;
+}
+
+.eval-toggle:hover {
+  color: #bbb;
+}
+
+.slo-badge {
+  font-size: 10px;
+  padding: 1px 6px;
+  border-radius: 3px;
+  font-weight: 600;
+}
+
+.slo-badge.pass {
+  background: #1b3a1b;
+  color: #4caf50;
+}
+
+.slo-badge.fail {
+  background: #3a1b1b;
+  color: #f44336;
+}
+
+.eval-details {
+  display: flex;
+  flex-direction: column;
+  gap: 10px;
+  margin-top: 8px;
+}
+
+.eval-section {
+  display: flex;
+  flex-direction: column;
+  gap: 4px;
+}
+
+.eval-na {
+  font-size: 11px;
+  color: #666;
+  font-style: italic;
+}
+
+.score-bar {
+  height: 4px;
+  background: #2a2d37;
+  border-radius: 2px;
+  overflow: hidden;
+}
+
+.score-bar-fill {
+  height: 100%;
+  border-radius: 2px;
+  transition: width 0.3s ease;
+}
+
+/* ── Star rating ─────────────────────────────────────────────────── */
+
+.star-rating {
+  display: flex;
+  align-items: center;
+  gap: 2px;
+}
+
+.star-rating .star {
+  background: none;
+  border: none;
+  font-size: 18px;
+  cursor: pointer;
+  color: #3a3d47;
+  padding: 0;
+  line-height: 1;
+  transition: color 0.15s;
+}
+
+.star-rating .star.active {
+  color: #ff9800;
+}
+
+.star-rating .star:hover {
+  color: #ffb74d;
+}
+
+.star-label {
+  font-size: 11px;
+  color: #888;
+  margin-left: 6px;
+}

frontend/src/components/ChatPanel.tsx

@@ -1,6 +1,7 @@
 import { useState, useRef, useEffect } from "react";
 import type { ChatMessage, SensingState, Affect, LatencyLog } from "../types";
 import { sendChat } from "../lib/api";
+import { EvalPanel } from "./EvalPanel";
 
 interface Props {
   userId: string | null;
@@ -59,6 +60,7 @@ export function ChatPanel({
           content: res.response,
           latency: res.latency,
           affect: res.affect,
+          evalScores: res.eval_scores,
         },
       ]);
       onLatency(res.latency);
@@ -88,6 +90,9 @@ export function ChatPanel({
               {msg.role === "partner" ? "Partner" : "AAC User"}
             </span>
             <p>{msg.content}</p>
+            {msg.role === "aac_user" && msg.evalScores && (
+              <EvalPanel evalScores={msg.evalScores} />
+            )}
           </div>
         ))}
         {loading && (

frontend/src/components/EvalPanel.tsx

@@ -0,0 +1,137 @@
+import { useState } from "react";
+import type { EvalScores } from "../types";
+
+interface Props {
+  evalScores: EvalScores;
+}
+
+function ScoreBar({ value }: { value: number }) {
+  const pct = Math.min(value * 100, 100);
+  const color = pct > 70 ? "#4caf50" : pct > 40 ? "#ff9800" : "#f44336";
+  return (
+    <div className="score-bar">
+      <div className="score-bar-fill" style={{ width: `${pct}%`, background: color }} />
+    </div>
+  );
+}
+
+function StarRating({
+  value,
+  onChange,
+}: {
+  value: number | null;
+  onChange: (v: number) => void;
+}) {
+  const [hover, setHover] = useState(0);
+  return (
+    <div className="star-rating">
+      {[1, 2, 3, 4, 5].map((star) => (
+        <button
+          key={star}
+          className={`star ${star <= (hover || (value ?? 0)) ? "active" : ""}`}
+          onMouseEnter={() => setHover(star)}
+          onMouseLeave={() => setHover(0)}
+          onClick={() => onChange(star)}
+        >
+          ★
+        </button>
+      ))}
+      {value !== null && <span className="star-label">{value}/5</span>}
+    </div>
+  );
+}
+
+export function EvalPanel({ evalScores }: Props) {
+  const [expanded, setExpanded] = useState(false);
+  const [likert, setLikert] = useState<number | null>(null);
+
+  return (
+    <div className="eval-panel">
+      <button
+        className="eval-toggle"
+        onClick={() => setExpanded(!expanded)}
+      >
+        {expanded ? "▾" : "▸"} Eval Metrics
+        {evalScores.slo_passed ? (
+          <span className="slo-badge pass">SLO ✓</span>
+        ) : (
+          <span className="slo-badge fail">SLO ✗</span>
+        )}
+        {likert !== null && (
+          <span className="slo-badge">{likert}/5 ★</span>
+        )}
+      </button>
+
+      {expanded && (
+        <div className="eval-details">
+          <div className="eval-section">
+            <div className="section-title">Factual Faithfulness</div>
+            {evalScores.no_evidence ? (
+              <div className="eval-na">N/A — no evidence retrieved</div>
+            ) : (
+              <>
+                <div className="metric-row">
+                  <span>Groundedness</span>
+                  <span className="metric-value">{(evalScores.groundedness * 100).toFixed(0)}%</span>
+                </div>
+                <ScoreBar value={evalScores.groundedness} />
+                <div className="metric-row">
+                  <span>Hallucination Rate</span>
+                  <span className={`metric-value ${evalScores.hallucination_rate > 0.2 ? "fail" : "pass"}`}>
+                    {(evalScores.hallucination_rate * 100).toFixed(0)}%
+                  </span>
+                </div>
+              </>
+            )}
+          </div>
+
+          <div className="eval-section">
+            <div className="section-title">Communication Efficiency</div>
+            <div className="metric-row">
+              <span>Response Time</span>
+              <span className={`metric-value ${evalScores.slo_passed ? "pass" : "fail"}`}>
+                {evalScores.t_total_s.toFixed(2)}s
+                {evalScores.slo_passed ? " ✓" : " ✗"}
+              </span>
+            </div>
+            <div className="metric-row sub">
+              <span>SLO Target</span>
+              <span className="metric-value">
+                &lt; {evalScores.slo_target_s.toFixed(1)}s (margin: {evalScores.slo_margin_s.toFixed(2)}s)
+              </span>
+            </div>
+          </div>
+
+          <div className="eval-section">
+            <div className="section-title">Multimodal Alignment</div>
+            <div className="metric-row">
+              <span>Overall</span>
+              <span className="metric-value">{(evalScores.multimodal_alignment * 100).toFixed(0)}%</span>
+            </div>
+            <ScoreBar value={evalScores.multimodal_alignment} />
+            <div className="metric-row sub">
+              <span>Affect</span>
+              <span className="metric-value">{(evalScores.affect_alignment * 100).toFixed(0)}%</span>
+            </div>
+            <div className="metric-row sub">
+              <span>Gesture</span>
+              <span className="metric-value">{(evalScores.gesture_alignment * 100).toFixed(0)}%</span>
+            </div>
+            <div className="metric-row sub">
+              <span>Gaze</span>
+              <span className="metric-value">{(evalScores.gaze_alignment * 100).toFixed(0)}%</span>
+            </div>
+          </div>
+
+          <div className="eval-section">
+            <div className="section-title">Perceived Authenticity</div>
+            <div className="metric-row">
+              <span>Rate this response</span>
+            </div>
+            <StarRating value={likert} onChange={setLikert} />
+          </div>
+        </div>
+      )}
+    </div>
+  );
+}

frontend/src/components/LatencyMetrics.tsx

@@ -16,8 +16,8 @@ export function LatencyMetrics({ latency }: Props) {
   if (!latency) return <p className="no-metrics">No turn yet</p>;
 
   return (
-    <div className="latency-metrics">
-      <h3>Latency</h3>
+    <div>
+      <div className="section-title">Latency</div>
       {FIELDS.map(({ key, label }) => (
         <div key={key} className="metric-row">
           <span className="metric-label">{label}</span>

frontend/src/types.ts

@@ -33,6 +33,20 @@ export interface LatencyLog {
   t_total: number;
 }
 
+export interface EvalScores {
+  groundedness: number;
+  hallucination_rate: number;
+  no_evidence: boolean;
+  t_total_s: number;
+  slo_target_s: number;
+  slo_passed: boolean;
+  slo_margin_s: number;
+  multimodal_alignment: number;
+  affect_alignment: number;
+  gesture_alignment: number;
+  gaze_alignment: number;
+}
+
 export interface ChatResponse {
   user_id: string;
   query: string;
@@ -42,6 +56,7 @@ export interface ChatResponse {
   retrieval_mode: string;
   latency: LatencyLog;
   guardrail_passed: boolean;
+  eval_scores: EvalScores | null;
 }
 
 export interface ChatMessage {
@@ -49,4 +64,5 @@ export interface ChatMessage {
   content: string;
   latency?: LatencyLog;
   affect?: string;
+  evalScores?: EvalScores | null;
 }

run.sh

@@ -3,6 +3,16 @@ set -euo pipefail
 
 export PYTHONWARNINGS="ignore::UserWarning:multiprocessing.resource_tracker"
 
+CONDA_ENV="aac-chatbot"
+
+# Activate conda env
+if ! command -v conda >/dev/null 2>&1; then
+  echo "ERROR: conda not found. Run setup.sh first." >&2
+  exit 1
+fi
+eval "$(conda shell.bash hook)"
+conda activate "$CONDA_ENV"
+
 PIDS=()
 
 cleanup() {
@@ -22,7 +32,7 @@ trap cleanup INT TERM
 
 # Use Node 22 if available (Vite 8 requires Node 20.19+ or 22.12+)
 if [ -x /opt/homebrew/opt/node@22/bin/node ]; then
-  export PATH="/opt/homebrew/opt/node@22/bin:$PATH"
+  export PATH="/opt/homebrew/opt/node@22/bin:/opt/homebrew/bin:$PATH"
 fi
 
 # Start Ollama if not already running