bug fixes

backend/evals/__init__.py

@@ -32,14 +32,12 @@ def _score_candidates_batched(
     else:
         relevances = [0.0] * len(texts)
 
-    scores = [{**faiths[i], "relevance": relevances[i]} for i in range(len(candidates))]
+    scores = [{**f, "relevance": r} for f, r in zip(faiths, relevances, strict=True)]
     return scores, cand_vecs
 
 
 def _diversity_from_vecs(cand_vecs: "torch.Tensor") -> dict:
     n = cand_vecs.shape[0]
-    if n < 2:
-        return {"candidate_diversity": 0.0, "n_candidates": n}
     sims = cand_vecs @ cand_vecs.T
     iu = torch.triu_indices(n, n, offset=1)
     return {
@@ -75,11 +73,11 @@ def compute_evals(
     per_cand: list[dict] = []
     cand_vecs = None
     if candidates:
-        if selected_idx is None and response:
-            for i, c in enumerate(candidates):
-                if c.get("text", "").strip() == response.strip():
-                    selected_idx = i
-                    break
+        # The planner serves uniq[0] as `selected_response`, so when caller
+        # didn't pass selected_idx explicitly, default to 0 rather than
+        # text-matching (which can collide on duplicate candidate texts).
+        if selected_idx is None:
+            selected_idx = 0
         scored, cand_vecs = _score_candidates_batched(candidates, chunks, query)
         per_cand = [
             {
@@ -117,11 +115,16 @@ def compute_evals(
 
     if per_cand:
         out["candidates_eval"] = per_cand
-        # Reuse cand_vecs from the relevance pass when available; falls back to
-        # standalone BGE encode (e.g. when query was empty).
-        if cand_vecs is not None:
+        n = len(candidates)
+        if n < 2:
+            out["candidate_diversity"] = 0.0
+            out["n_candidates"] = n
+        elif cand_vecs is not None:
+            # Reuse vectors from the relevance pass.
             out.update(_diversity_from_vecs(cand_vecs))
         else:
+            # Standalone BGE encode (e.g. when query was empty so the relevance
+            # pass was skipped).
             out.update(compute_candidate_diversity(candidates))
     else:
         out["candidate_diversity"] = 0.0

backend/evals/aggregate.py

@@ -11,6 +11,9 @@ from pathlib import Path
 
 from backend.config.settings import settings
 
+# Mean pairwise cosine distance below this means the picker showed near-paraphrases.
+_DIVERSITY_FLOOR = 0.10
+
 
 def _load(path: Path) -> list[dict]:
     if not path.exists():
@@ -256,11 +259,11 @@ def report_picker(turns: list[dict], picks: list[dict], evals: list[dict]) -> No
     ]
     if div_scored:
         diversities = [float(e["candidate_diversity"]) for e in div_scored]
-        low = sum(1 for d in diversities if d < 0.1)
+        low = sum(1 for d in diversities if d < _DIVERSITY_FLOOR)
         print(
             f"\nCandidate diversity (n={len(div_scored)} turns): "
             f"mean={statistics.mean(diversities):.2f}  "
-            f"low (<0.10): {low}/{len(div_scored)} ({low / len(div_scored):.0%})"
+            f"low (<{_DIVERSITY_FLOOR:.2f}): {low}/{len(div_scored)} ({low / len(div_scored):.0%})"
         )
 
 

backend/evals/diversity.py

@@ -4,7 +4,9 @@ import torch
 def compute_candidate_diversity(candidates: list[dict]) -> dict:
     """Mean pairwise cosine *distance* among candidate texts.
 
-    1.0 = maximally different, 0.0 = identical paraphrases.
+    1.0 = maximally different, 0.0 = identical paraphrases. Empty candidate
+    texts are filtered out before encoding, so `n_candidates` in the result
+    is the count of *non-empty* texts (may be < len(candidates)).
     """
     texts = [c.get("text", "").strip() for c in candidates]
     texts = [t for t in texts if t]

backend/evals/relevance.py

@@ -5,12 +5,9 @@ def compute_relevance(response: str, query: str) -> dict:
     its answer), so we use the same embedding space the retriever uses.
     """
     if not response.strip() or not query.strip():
-        return {"relevance": 0.0, "no_query": not query.strip()}
+        return {"relevance": 0.0}
 
     from backend.retrieval.vector_store import embed_texts
 
     vecs = embed_texts([query, response])
-    return {
-        "relevance": round(max(0.0, float(vecs[0] @ vecs[1])), 4),
-        "no_query": False,
-    }
+    return {"relevance": round(max(0.0, float(vecs[0] @ vecs[1])), 4)}

frontend/src/components/EvalPanel.tsx

@@ -265,23 +265,21 @@ function EvalPanelImpl({
             )}
           </>
         )}
-        <span className="tip" data-tip="Rate how authentic this response felt as the persona (1 = off, 5 = spot on). Logged to ratings.jsonl.">
-          <div className="star-rating">
-            {[1, 2, 3, 4, 5].map((star) => (
-              <button
-                key={star}
-                className={`star ${star <= (hover || (value ?? 0)) ? "active" : ""}`}
-                onMouseEnter={() => setHover(star)}
-                onMouseLeave={() => setHover(0)}
-                onClick={() => rate(star)}
-                disabled={value !== null || submitting}
-              >
-                ★
-              </button>
-            ))}
-            {value !== null && <span className="star-label">{value}/5</span>}
-          </div>
-        </span>
+        <div className="tip star-rating" data-tip="Rate how authentic this response felt as the persona (1 = off, 5 = spot on). Logged to ratings.jsonl.">
+          {[1, 2, 3, 4, 5].map((star) => (
+            <button
+              key={star}
+              className={`star ${star <= (hover || (value ?? 0)) ? "active" : ""}`}
+              onMouseEnter={() => setHover(star)}
+              onMouseLeave={() => setHover(0)}
+              onClick={() => rate(star)}
+              disabled={value !== null || submitting}
+            >
+              ★
+            </button>
+          ))}
+          {value !== null && <span className="star-label">{value}/5</span>}
+        </div>
       </div>
     </div>
   );