Harden ingestion and retrieval reliability across the pipeline

ARCHITECTURE.md

@@ -133,7 +133,7 @@ morpheus/
 
 | Function | Purpose |
 |----------|---------|
-| `hybrid_search(query_text, query_embedding, match_count, filter, semantic_weight, keyword_weight)` | Combined BM25 + pgvector search |
+| `hybrid_search(query_text, query_embedding, match_count, filter, semantic_weight, keyword_weight, p_user_id)` | Combined BM25 + pgvector search (tenant-scoped overload) |
 | `match_memory(query_embedding, match_session_id, match_count)` | Semantic search over chat history |
 | `insert_document_chunk(p_id, p_content, p_metadata, p_embedding, p_user_id)` | Secure insert with explicit user_id |
 | `get_document_types()` | Returns distinct categories for this tenant |
@@ -221,6 +221,12 @@ Step 1: Intent analysis (analyse_intent)
   Reference queries ("summarise it"): replaced with previous query
   Every query logged to intent_feedback for online retraining
 
+Step 1.5: Ambiguity / scope safety (check_query_ambiguity)
+  If the user has NOT pinned a document:
+  - If **multiple docs are in scope** and the query is **identity/page-scoped** (owner/title/publisher/cover/first page), Morpheus **asks the user to pick a document** (never guesses).
+  - Otherwise, Morpheus may ask a clarification question for generic queries when multiple docs match.
+  Implementation detail: ambiguity scoring uses `hybrid_search(..., p_user_id=...)` to avoid PostgREST overload ambiguity.
+
 Step 2: Query routing
   Structural queries (table of contents, numbered items, specific codes)?
     → tree_search(): recursive traversal of document_trees for this user
@@ -231,6 +237,8 @@ Step 3: retrieve_chunks() — vector path
   a) Follow-up detection
      Query ≤8 words with pronouns (it/this/that/they)?
      Reuse _last_chunks[session_key] — no re-search
+     Safety guard: ordinal follow-ups like "the second one" must have an explicit referent (a list);
+     otherwise the API asks for clarification instead of guessing.
 
   b) Semantic cache check
      Embed query (256-entry in-memory LRU cache)

backend/api/admin.py

@@ -1,11 +1,16 @@
 """backend/api/admin.py — Admin endpoints, protected by X-Admin-Key header."""
 
 import os, hmac, logging  # noqa: E401
+from datetime import datetime, timedelta, timezone
+from collections import Counter
+from typing import Optional
+
 from fastapi import APIRouter, HTTPException, Header, Depends
+from pydantic import BaseModel
+
 from backend.core.auth_utils import require_auth_token
 from backend.core.warmup_classifier import warmup, warmup_cross_encoder
-from datetime import datetime, timedelta, timezone
-from collections import Counter
+from backend.core.pipeline import _build_service_supabase_client
 
 log = logging.getLogger("morpheus.api.admin")
 router = APIRouter()
@@ -19,6 +24,78 @@ def _check_admin(key: str):
         raise HTTPException(status_code=403, detail="Invalid admin key.")
 
 
+class ReviewPayload(BaseModel):
+    review_state: str = "reviewed"
+    review_notes: Optional[str] = None
+
+
+def _admin_client():
+    return _build_service_supabase_client()
+
+
+def _trace_sort_key(row: dict):
+    return row.get("created_at") or ""
+
+
+def _feedback_sort_key(row: dict):
+    return row.get("created_at") or ""
+
+
+def _load_recent_traces(*, limit: int = 100) -> list[dict]:
+    rows = (
+        _admin_client()
+        .table("query_traces")
+        .select(
+            "trace_id, question, route_mode, selected_experts, expert_weights, "
+            "document_types, doc_diagnostics, failure_modes, quality_metrics, "
+            "answer_preview, latency_ms, review_state, review_notes, reviewed_at, "
+            "reviewed_by, promoted_to_eval, created_at"
+        )
+        .limit(limit)
+        .execute()
+        .data
+        or []
+    )
+    return sorted(rows, key=_trace_sort_key, reverse=True)
+
+
+def _load_recent_feedback(*, limit: int = 100) -> list[dict]:
+    rows = (
+        _admin_client()
+        .table("answer_feedback")
+        .select(
+            "id, trace_id, helpful, accepted, reason_code, correction_text, "
+            "promote_to_eval, review_state, review_notes, reviewed_at, reviewed_by, "
+            "promoted_at, created_at, user_id"
+        )
+        .limit(limit)
+        .execute()
+        .data
+        or []
+    )
+    return sorted(rows, key=_feedback_sort_key, reverse=True)
+
+
+def _build_eval_dataset_row(trace_row: dict, feedback_row: dict) -> dict:
+    correction_text = (feedback_row.get("correction_text") or "").strip()
+    answer_preview = (trace_row.get("answer_preview") or "").strip()
+    return {
+        "trace_id": trace_row.get("trace_id"),
+        "source": "feedback_trace",
+        "question": trace_row.get("question"),
+        "gold_context_refs": [],
+        "gold_evidence_text": correction_text or answer_preview,
+        "is_answerable": bool(
+            feedback_row.get("accepted")
+            or feedback_row.get("helpful")
+        ),
+        "failure_modes": trace_row.get("failure_modes") or [],
+        "doc_diagnostics": trace_row.get("doc_diagnostics") or [],
+        "reason_code": feedback_row.get("reason_code"),
+        "is_active": False,
+    }
+
+
 @router.post("/warmup")
 def run_warmup(x_admin_key: str = Header(..., alias="X-Admin-Key")):
     _check_admin(x_admin_key)
@@ -105,4 +182,210 @@ def get_corpus_health(
         "recommendation": "Prompt user to upload documents regarding content gaps."
         if missing_topics
         else "Corpus coverage is sufficient.",
-    }
+    }
+
+
+@router.get("/traces")
+def list_query_traces(
+    x_admin_key: str = Header(..., alias="X-Admin-Key"),
+    limit: int = 50,
+    route_mode: Optional[str] = None,
+    failure_mode: Optional[str] = None,
+    category: Optional[str] = None,
+    hours: int = 168,
+    review_state: Optional[str] = None,
+):
+    _check_admin(x_admin_key)
+    traces = _load_recent_traces(limit=max(limit * 3, 100))
+    cutoff = datetime.now(timezone.utc) - timedelta(hours=max(1, hours))
+    filtered = []
+    for row in traces:
+        created_raw = row.get("created_at")
+        created_at = None
+        if isinstance(created_raw, str):
+            try:
+                created_at = datetime.fromisoformat(created_raw.replace("Z", "+00:00"))
+            except Exception:
+                created_at = None
+        if created_at and created_at < cutoff:
+            continue
+        if route_mode and row.get("route_mode") != route_mode:
+            continue
+        if failure_mode and failure_mode not in (row.get("failure_modes") or []):
+            continue
+        if review_state and row.get("review_state") != review_state:
+            continue
+        if category and category not in (row.get("document_types") or []):
+            continue
+        filtered.append(row)
+    return {"items": filtered[:limit]}
+
+
+@router.get("/traces/{trace_id}")
+def get_query_trace(
+    trace_id: str,
+    x_admin_key: str = Header(..., alias="X-Admin-Key"),
+):
+    _check_admin(x_admin_key)
+    sb = _admin_client()
+    trace_rows = (
+        sb.table("query_traces")
+        .select("*")
+        .eq("trace_id", trace_id)
+        .limit(1)
+        .execute()
+        .data
+        or []
+    )
+    if not trace_rows:
+        raise HTTPException(status_code=404, detail="Trace not found.")
+    feedback_rows = (
+        sb.table("answer_feedback")
+        .select("*")
+        .eq("trace_id", trace_id)
+        .execute()
+        .data
+        or []
+    )
+    return {"trace": trace_rows[0], "feedback": sorted(feedback_rows, key=_feedback_sort_key, reverse=True)}
+
+
+@router.post("/traces/{trace_id}/review")
+def review_query_trace(
+    trace_id: str,
+    payload: ReviewPayload,
+    x_admin_key: str = Header(..., alias="X-Admin-Key"),
+):
+    _check_admin(x_admin_key)
+    now_iso = datetime.now(timezone.utc).isoformat()
+    _admin_client().table("query_traces").update(
+        {
+            "review_state": payload.review_state,
+            "review_notes": payload.review_notes,
+            "reviewed_at": now_iso,
+            "reviewed_by": "admin",
+        }
+    ).eq("trace_id", trace_id).execute()
+    return {"ok": True}
+
+
+@router.get("/feedback")
+def list_feedback(
+    x_admin_key: str = Header(..., alias="X-Admin-Key"),
+    limit: int = 50,
+    review_state: Optional[str] = None,
+    promote_only: bool = False,
+):
+    _check_admin(x_admin_key)
+    rows = _load_recent_feedback(limit=max(limit * 3, 100))
+    filtered = []
+    for row in rows:
+        if review_state and row.get("review_state") != review_state:
+            continue
+        if promote_only and not row.get("promote_to_eval"):
+            continue
+        filtered.append(row)
+    return {"items": filtered[:limit]}
+
+
+@router.get("/feedback/{feedback_id}")
+def get_feedback_detail(
+    feedback_id: int,
+    x_admin_key: str = Header(..., alias="X-Admin-Key"),
+):
+    _check_admin(x_admin_key)
+    sb = _admin_client()
+    rows = (
+        sb.table("answer_feedback")
+        .select("*")
+        .eq("id", feedback_id)
+        .limit(1)
+        .execute()
+        .data
+        or []
+    )
+    if not rows:
+        raise HTTPException(status_code=404, detail="Feedback not found.")
+    feedback = rows[0]
+    trace_rows = (
+        sb.table("query_traces")
+        .select("*")
+        .eq("trace_id", feedback.get("trace_id"))
+        .limit(1)
+        .execute()
+        .data
+        or []
+    )
+    return {"feedback": feedback, "trace": trace_rows[0] if trace_rows else None}
+
+
+@router.post("/feedback/{feedback_id}/review")
+def review_feedback(
+    feedback_id: int,
+    payload: ReviewPayload,
+    x_admin_key: str = Header(..., alias="X-Admin-Key"),
+):
+    _check_admin(x_admin_key)
+    now_iso = datetime.now(timezone.utc).isoformat()
+    _admin_client().table("answer_feedback").update(
+        {
+            "review_state": payload.review_state,
+            "review_notes": payload.review_notes,
+            "reviewed_at": now_iso,
+            "reviewed_by": "admin",
+        }
+    ).eq("id", feedback_id).execute()
+    return {"ok": True}
+
+
+@router.post("/feedback/{feedback_id}/promote")
+def promote_feedback_to_eval(
+    feedback_id: int,
+    x_admin_key: str = Header(..., alias="X-Admin-Key"),
+):
+    _check_admin(x_admin_key)
+    sb = _admin_client()
+    feedback_rows = (
+        sb.table("answer_feedback")
+        .select("*")
+        .eq("id", feedback_id)
+        .limit(1)
+        .execute()
+        .data
+        or []
+    )
+    if not feedback_rows:
+        raise HTTPException(status_code=404, detail="Feedback not found.")
+    feedback = feedback_rows[0]
+    trace_rows = (
+        sb.table("query_traces")
+        .select("*")
+        .eq("trace_id", feedback.get("trace_id"))
+        .limit(1)
+        .execute()
+        .data
+        or []
+    )
+    if not trace_rows:
+        raise HTTPException(status_code=404, detail="Trace not found.")
+    trace = trace_rows[0]
+    row = _build_eval_dataset_row(trace, feedback)
+    sb.table("evaluation_datasets").upsert(row, on_conflict="trace_id").execute()
+    now_iso = datetime.now(timezone.utc).isoformat()
+    sb.table("answer_feedback").update(
+        {
+            "review_state": "promoted",
+            "promoted_at": now_iso,
+            "reviewed_at": now_iso,
+            "reviewed_by": "admin",
+        }
+    ).eq("id", feedback_id).execute()
+    sb.table("query_traces").update(
+        {
+            "review_state": "promoted",
+            "promoted_to_eval": True,
+            "reviewed_at": now_iso,
+            "reviewed_by": "admin",
+        }
+    ).eq("trace_id", trace.get("trace_id")).execute()
+    return {"ok": True}

backend/api/auth.py

@@ -7,10 +7,11 @@ declare `auth: AuthContext = Depends(require_auth)` — see the pattern
 at the bottom of this file and replicate it in each router.
 """
 
-from fastapi import APIRouter, Depends
+from fastapi import APIRouter, Depends, Header, HTTPException
 
-from backend.core.auth_utils import require_auth_token
-from backend.services.auth import get_daily_password, verify_admin_key, verify_password
+from backend.core.auth_utils import is_guest_token, require_auth_token
+from backend.core.pipeline import _build_service_supabase_client
+from backend.services.auth import verify_admin_key, verify_password
 from shared.types import AuthRequest, AuthResponse
 
 router = APIRouter()
@@ -31,7 +32,7 @@ def verify(req: AuthRequest):
 @router.post("/admin", response_model=AuthResponse)
 def admin_verify(req: AuthRequest):
     if verify_admin_key(req.password):
-        return AuthResponse(valid=True, token=get_daily_password(), message="Admin verified.")
+        return AuthResponse(valid=True, message="Admin verified.")
     return AuthResponse(valid=False, message="Invalid admin key.")
 
 
@@ -40,3 +41,73 @@ def admin_verify(req: AuthRequest):
 async def get_me(user_id: str = Depends(require_auth_token)):
     return {"user_id": user_id, "authenticated": True}
 
+
+@router.delete("/guest-workspace")
+async def clear_guest_workspace(
+    user_id: str = Depends(require_auth_token),
+    x_auth_token: str = Header(None, alias="X-Auth-Token"),
+):
+    if not is_guest_token(x_auth_token):
+        raise HTTPException(status_code=403, detail="Guest workspace cleanup is only for guest sessions.")
+
+    sb = _build_service_supabase_client()
+
+    # Preserve anonymized adaptive signals while removing the guest's actual workspace.
+    try:
+        sb.table("query_traces").update(
+            {
+                "user_id": None,
+                "session_id": "guest_archived",
+                "question": "[guest session removed]",
+                "pinned_file_hashes": [],
+                "selected_chunk_ids": [],
+                "doc_diagnostics": [],
+                "answer_preview": None,
+                "document_types": [],
+            }
+        ).eq("user_id", user_id).execute()
+    except Exception:
+        pass
+
+    try:
+        sb.table("answer_feedback").update(
+            {
+                "user_id": None,
+                "correction_text": None,
+            }
+        ).eq("user_id", user_id).execute()
+    except Exception:
+        pass
+
+    try:
+        sb.table("evaluation_logs").update(
+            {
+                "user_id": None,
+                "question": "[guest session removed]",
+            }
+        ).eq("user_id", user_id).execute()
+    except Exception:
+        pass
+
+    def _purge(table_name: str) -> None:
+        try:
+            sb.table(table_name).delete().eq("user_id", user_id).execute()
+        except Exception:
+            # Optional/older tables should not break guest cleanup.
+            pass
+
+    # Delete child/content tables first, then registry-ish tables.
+    for table_name in (
+        "documents",
+        "document_trees",
+        "chat_memory",
+        "ingestion_retry_logs",
+        "rerank_feedback",
+        "intent_feedback",
+        "graph_edges",
+        "graph_nodes",
+        "ingested_files",
+    ):
+        _purge(table_name)
+
+    return {"ok": True, "message": "Guest workspace cleared."}

backend/api/frontend_config.py

@@ -1,4 +1,4 @@
-from fastapi import APIRouter
+from fastapi import APIRouter, HTTPException
 from backend.core import config
 
 router = APIRouter()
@@ -9,7 +9,13 @@ def get_frontend_config():
     Returns public config values the frontend needs.
     Only exposes the anon key (safe by design) — never the service key.
     """
+    if not config.SUPABASE_URL or not config.SUPABASE_ANON_KEY:
+        raise HTTPException(
+            status_code=503,
+            detail="Supabase frontend config is missing on the server.",
+        )
     return {
         "supabase_url":  config.SUPABASE_URL,
         "supabase_anon": config.SUPABASE_ANON_KEY,
-    }
+        "guest_enabled": config.GUEST_MODE_ENABLED,
+    }

backend/api/ingest.py

@@ -1,8 +1,10 @@
 import os
 import tempfile
 import logging
-from fastapi import APIRouter, UploadFile, File, HTTPException, Header, Depends
-from backend.core.auth_utils import require_auth_token
+from fastapi import APIRouter, UploadFile, File, HTTPException, Header, Depends, Request
+from backend.core import config
+from backend.core.auth_utils import is_guest_token, require_auth_token
+from backend.core.rate_limit import limiter
 from backend.core.tasks import process_pdf_task
 from backend.core.tasks import celery_app
 
@@ -10,15 +12,39 @@ log = logging.getLogger("morpheus.api.ingest")
 router = APIRouter()
 
 
+def _cleanup_temp_upload(tmp_path: str) -> None:
+    if not tmp_path:
+        return
+    try:
+        os.unlink(tmp_path)
+    except FileNotFoundError:
+        return
+    except OSError as exc:
+        log.warning("Could not remove temp upload %s: %s", tmp_path, exc)
+
+
+def _ensure_ingest_worker_available() -> None:
+    if celery_app is None or not hasattr(process_pdf_task, "delay"):
+        raise HTTPException(
+            status_code=503,
+            detail="Background ingestion worker is unavailable.",
+        )
+
+
 @router.post("/upload")
+@limiter.limit("12/hour")
 async def upload(
+    request: Request,
     file: UploadFile = File(...),
     user_id: str = Depends(require_auth_token),
     x_auth_token: str = Header(None, alias="X-Auth-Token"),
 ):
+    del request
     if not file.filename.lower().endswith(".pdf"):
         raise HTTPException(status_code=400, detail="Only PDF files are supported.")
 
+    guest_workspace = is_guest_token(x_auth_token)
+
     # NEW: Secure file signature validation using python-magic
     import magic
 
@@ -33,6 +59,8 @@ async def upload(
         )
 
     # ── Per-user document limit ───────────────────────────────────────────────
+    doc_limit = config.GUEST_MAX_DOCS if guest_workspace else config.MAX_DOCS_PER_USER
+
     try:
         from backend.core.pipeline import _build_supabase_client
 
@@ -43,20 +71,33 @@ async def upload(
             .eq("user_id", user_id)
             .execute()
         )
-        if (result.count or 0) >= 50:
+        if (result.count or 0) >= doc_limit:
             raise HTTPException(
-                status_code=429, detail="Document limit reached (50 max)."
+                status_code=429,
+                detail=f"Document limit reached ({doc_limit} max).",
             )
     except HTTPException:
         raise
-    except Exception:
-        pass  # don't block upload if count check fails
+    except Exception as exc:
+        log.error("Upload limit check failed for user %s: %s", user_id, exc)
+        raise HTTPException(
+            status_code=503,
+            detail="Could not verify upload limits right now. Please try again.",
+        ) from exc
+
+    _ensure_ingest_worker_available()
 
     # Safely save to disk as before
     tmp_fd, tmp_path = tempfile.mkstemp(suffix=f"_{file.filename}")
     os.close(tmp_fd)  # close fd immediately, manage file separately
     try:
         contents = await file.read()
+        max_upload_mb = config.GUEST_MAX_UPLOAD_MB if guest_workspace else config.MAX_UPLOAD_MB
+        if len(contents) > max_upload_mb * 1024 * 1024:
+            raise HTTPException(
+                status_code=413,
+                detail=f"File too large ({max_upload_mb} MB max).",
+            )
         with open(tmp_path, "wb") as f:
             f.write(contents)
         task = process_pdf_task.delay(tmp_path, file.filename, x_auth_token)
@@ -65,18 +106,19 @@ async def upload(
             "task_id": task.id,
             "filename": file.filename,
         }
+    except HTTPException:
+        _cleanup_temp_upload(tmp_path)
+        raise
     except Exception as e:
-        log.error("Failed to queue file: %s", e)
-        try:
-            os.unlink(tmp_path)
-        except OSError:
-            pass
-        raise HTTPException(status_code=500, detail="Failed to queue file.")
+        log.exception("Failed to queue file: %s", e)
+        _cleanup_temp_upload(tmp_path)
+        raise HTTPException(status_code=500, detail="Failed to queue file.") from e
 
 
 # NEW ROUTE: The frontend will poll this every 2 seconds
 @router.get("/status/{task_id}")
 def get_ingest_status(task_id: str):
+    _ensure_ingest_worker_available()
     task_result = celery_app.AsyncResult(task_id)
 
     if task_result.state == "PENDING":

backend/api/query.py

@@ -2,21 +2,58 @@
 import json
 import logging
 import asyncio
-from fastapi import APIRouter, Header, Depends, Request
+from fastapi import APIRouter, Header, Depends, Request, HTTPException
 from fastapi.responses import StreamingResponse
-from shared.types import QueryRequest, SourceChunk
+from shared.types import AnswerFeedback, QueryRequest, SourceChunk
 from backend.core.pipeline import (
     retrieve_chunks_routed,
     generate_answer_stream,
     analyse_intent,
+    check_query_ambiguity,
+    record_answer_feedback,
 )
 from backend.core.auth_utils import require_auth_token
-from backend.main import limiter
+from backend.core.rate_limit import limiter
 
 log = logging.getLogger("morpheus.api.query")
 router = APIRouter()
 
 
+def _contains_ordinal_followup(query: str) -> bool:
+    q = (query or "").strip().lower()
+    if not q:
+        return False
+    return any(
+        phrase in q
+        for phrase in (
+            "the second one",
+            "the first one",
+            "the other one",
+            "second one",
+            "first one",
+            "other one",
+        )
+    )
+
+
+def _history_has_explicit_enumeration(history: list[dict]) -> bool:
+    """
+    Heuristic: if the last assistant message contains an explicit list, then
+    ordinal follow-ups (\"second one\") can be resolved. Otherwise, ask.
+    """
+    for msg in reversed(history or []):
+        if (msg.get("role") or "").lower() != "assistant":
+            continue
+        content = str(msg.get("content") or "")
+        if not content.strip():
+            return False
+        # Common enumeration patterns (numbers, bullets).
+        if any(token in content for token in ("\n1.", "\n2.", "\n- ", "\n• ")):
+            return True
+        return False
+    return False
+
+
 def _normalise_original_content(raw):
     """Best-effort decode for metadata that may already be dict or JSON string."""
     if isinstance(raw, dict):
@@ -91,14 +128,68 @@ async def query(
                 user_id      = user_id,
             )
 
+            if intent.get("route_class") == "no_retrieval":
+                yield "data: " + json.dumps({
+                    "type": "token",
+                    "content": "Ask me about your uploaded documents or a topic inside them, and I’ll dig in.",
+                }) + "\n\n"
+                yield "data: " + json.dumps({
+                    "type": "done",
+                    "sources": [],
+                    "images": [],
+                    "trace_id": None,
+                    "doc_diagnostics": [],
+                }) + "\n\n"
+                return
+
             if not intent.get("is_clear"):
                 # Stream clarification question as a normal assistant message
                 # User answers it → next turn history resolves the subject
                 question = intent.get("clarification_question", "Could you clarify?")
                 yield "data: " + json.dumps({"type": "token",  "content": question}) + "\n\n"
-                yield "data: " + json.dumps({"type": "done", "sources": [], "images": []}) + "\n\n"
+                yield "data: " + json.dumps({"type": "done", "sources": [], "images": [], "trace_id": None, "doc_diagnostics": []}) + "\n\n"
+                return
+
+            # Guardrail: ordinal follow-ups without an explicit referent should not guess.
+            if (
+                intent.get("route_class") == "follow_up"
+                and _contains_ordinal_followup(req.query)
+                and not _history_has_explicit_enumeration(history)
+            ):
+                yield "data: " + json.dumps(
+                    {
+                        "type": "token",
+                        "content": "Second one of what? Please reference the items you mean (e.g., paste the list or restate the names).",
+                    }
+                ) + "\n\n"
+                yield "data: " + json.dumps(
+                    {"type": "done", "sources": [], "images": [], "trace_id": None, "doc_diagnostics": []}
+                ) + "\n\n"
                 return
 
+            # ── Step 1.5: Phase 2 Ambiguity Detection ────────────────────────
+            # If no manual pin is active, check if the query is too ambiguous
+            if not req.priority_file_hashes:
+                ambiguity_res = check_query_ambiguity(
+                    req.query,
+                    access_token=x_auth_token,
+                    category=req.category,
+                )
+                if ambiguity_res.get("is_ambiguous"):
+                    question = ambiguity_res.get("clarification_question", "Which document do you mean?")
+                    # Use a distinct identifier so the frontend understands it's a structural prompt
+                    yield "data: " + json.dumps({"type": "token", "content": question}) + "\n\n"
+                    
+                    options = ambiguity_res.get("clarification_options")
+                    if options:
+                        yield "data: " + json.dumps({"type": "clarification_options", "options": options}) + "\n\n"
+
+                    yield "data: " + json.dumps({"type": "done", "sources": [], "images": [], "trace_id": None, "doc_diagnostics": []}) + "\n\n"
+                    return
+                if ambiguity_res.get("top_file_hash") and not ambiguity_res.get("is_ambiguous"):
+                    req.priority_file_hashes = [ambiguity_res["top_file_hash"]]
+                    log.info("Auto-pinned file hash: %s", ambiguity_res["top_file_hash"])
+
             # ── Step 2: Retrieve using enriched query ─────────────────────────
             # enriched_query has better embedding signal (category/history injected)
             # but we answer with the ORIGINAL query so the response sounds natural
@@ -117,12 +208,15 @@ async def query(
                     user_id=user_id,
                     original_query=req.query,
                     eval_mode=(x_eval_mode == "true"),
+                    priority_file_hashes=req.priority_file_hashes or None,
                 ),
             )
 
             # ── Step 3: Stream answer tokens ──────────────────────────────────
             images = []
             done_sources = []
+            trace_id = None
+            doc_diagnostics = []
             # 🚀 Define the boolean once for readability
             is_eval = x_eval_mode == "true"
             async for event in generate_answer_stream(
@@ -133,12 +227,15 @@ async def query(
                 access_token=x_auth_token,
                 category=category,
                 eval_mode=is_eval,
+                priority_file_hashes=req.priority_file_hashes or None,
             ):
                 if event["type"] == "token":
                     yield "data: " + json.dumps({"type": "token", "content": event["content"]}) + "\n\n"
                 elif event["type"] == "done":
                     images = event.get("images", [])
                     done_sources = event.get("sources", []) or []
+                    trace_id = event.get("trace_id")
+                    doc_diagnostics = event.get("doc_diagnostics", []) or []
 
             # ── Step 4: Emit sources + images ─────────────────────────────────
             sources = done_sources or _build_sources_from_chunks(
@@ -149,6 +246,8 @@ async def query(
                 "type":    "done",
                 "sources": sources,
                 "images":  images,
+                "trace_id": trace_id,
+                "doc_diagnostics": doc_diagnostics,
             }) + "\n\n"
 
         except Exception as e:
@@ -178,3 +277,16 @@ async def query(
             "Access-Control-Allow-Origin": "*",
         }
     )
+
+
+@router.post("/feedback")
+async def submit_feedback(
+    payload: AnswerFeedback,
+    user_id: str = Depends(require_auth_token),
+    x_auth_token: str = Header(None, alias="X-Auth-Token"),
+):
+    del user_id
+    ok = record_answer_feedback(payload.dict(), access_token=x_auth_token)
+    if not ok:
+        raise HTTPException(status_code=500, detail="Could not record answer feedback.")
+    return {"ok": True}

backend/core/auth_utils.py

@@ -12,7 +12,7 @@ TASK 1 — Auth Bridge:
 
 import jwt
 import logging
-from typing import Optional
+from typing import Any, Optional
 from backend.core import config
 from fastapi import Header, HTTPException, status
 
@@ -22,6 +22,45 @@ from fastapi import Header, HTTPException, status
 log = logging.getLogger("morpheus.auth")
 
 
+def _decode_unverified_claims(access_token: Optional[str]) -> dict[str, Any]:
+    """Peek at JWT claims without verifying the signature for non-security decisions."""
+    if not access_token:
+        return {}
+    try:
+        claims = jwt.decode(
+            access_token,
+            options={
+                "verify_signature": False,
+                "verify_exp": False,
+                "verify_aud": False,
+            },
+            algorithms=["ES256", "HS256", "RS256"],
+        )
+        return claims if isinstance(claims, dict) else {}
+    except Exception:
+        return {}
+
+
+def is_guest_token(access_token: Optional[str]) -> bool:
+    """
+    Supabase anonymous users still get real JWTs.
+    We treat them as guest workspaces for UI/limits/rate-limiting.
+    """
+    claims = _decode_unverified_claims(access_token)
+    if not claims:
+        return False
+
+    app_meta = claims.get("app_metadata") or {}
+    provider = str(app_meta.get("provider") or "").strip().lower()
+    providers = app_meta.get("providers") or []
+    return bool(
+        claims.get("is_anonymous")
+        or app_meta.get("is_anonymous")
+        or provider == "anonymous"
+        or "anonymous" in providers
+    )
+
+
 def extract_jwt_sub(access_token: str) -> str:
     """
     Extract the Supabase user id (JWT `sub`) while strictly verifying the signature.

backend/core/classifier.py

@@ -167,8 +167,9 @@ class CentroidStore:
         self._access_token = access_token
         self._user_id = None
         if access_token:
-            from backend.core.auth_utils import extract_jwt_sub
-            self._user_id = extract_jwt_sub(access_token)
+            from backend.core.auth_utils import safe_extract_jwt_sub
+
+            self._user_id = safe_extract_jwt_sub(access_token)
         self._cache: Dict[str, Dict] = {}
         self._lock   = threading.Lock()
         self._client = None
@@ -176,23 +177,17 @@ class CentroidStore:
 
     def _get_client(self):
         if self._client is None:
-            # Tenant-scoped client (anon + access token) is required for RLS isolation.
-            if self._access_token:
-                if not config.SUPABASE_ANON_KEY:
-                    raise RuntimeError("SUPABASE_ANON_KEY is not set but access_token was provided.")
-                self._client = create_client(
-                    config.SUPABASE_URL,
-                    config.SUPABASE_ANON_KEY,
-                )
-                self._client.postgrest.auth(self._access_token)
-            else:
-                # Admin / legacy fallback (bypasses RLS via service role).
-                self._client = create_client(config.SUPABASE_URL, config.SUPABASE_SERVICE_KEY)
+            # Backend-owned access model: always use the service-role client and
+            # scope rows explicitly by user_id where applicable.
+            self._client = create_client(config.SUPABASE_URL, config.SUPABASE_SERVICE_KEY)
         return self._client
 
     def _load_from_db(self):
         try:
-            result = self._get_client().table(self.TABLE).select("*").execute()
+            query = self._get_client().table(self.TABLE).select("*")
+            if self._user_id:
+                query = query.eq("user_id", self._user_id)
+            result = query.execute()
             for row in (result.data or []):
                 self._cache[row["document_type"]] = {
                     "vector": np.array(row["centroid_vector"], dtype=np.float32),

backend/core/config.py

@@ -19,6 +19,15 @@ SUPABASE_SERVICE_KEY = os.getenv("SUPABASE_SERVICE_KEY")
 SUPABASE_JWT_SECRET = os.getenv("SUPABASE_JWT_SECRET")
 VECTOR_TABLE_NAME = "documents"
 IMAGE_STORAGE_BUCKET = "rag-images"
+GUEST_MODE_ENABLED = os.getenv("GUEST_MODE_ENABLED", "true").lower() in {
+    "1",
+    "true",
+    "yes",
+}
+MAX_UPLOAD_MB = int(os.getenv("MAX_UPLOAD_MB", "25"))
+GUEST_MAX_UPLOAD_MB = int(os.getenv("GUEST_MAX_UPLOAD_MB", "10"))
+MAX_DOCS_PER_USER = int(os.getenv("MAX_DOCS_PER_USER", "50"))
+GUEST_MAX_DOCS = int(os.getenv("GUEST_MAX_DOCS", "10"))
 
 # ==================== API KEYS ====================
 OPENROUTER_API_KEY = os.getenv("OPENROUTER_API_KEY")
@@ -37,9 +46,19 @@ OLLAMA_MODELS = ["llama3.2", "mistral"]
 EMBEDDING_MODEL = "nvidia/llama-nemotron-embed-vl-1b-v2:free"
 EMBEDDING_DIMENSIONS = 2048
 EMBEDDING_DEVICE = "cuda"
+RETRIEVAL_EMBEDDING_VARIANT = os.getenv(
+    "RETRIEVAL_EMBEDDING_VARIANT", "control"
+).strip().lower()
+RETRIEVAL_EMBEDDING_MODEL_OVERRIDE = os.getenv(
+    "RETRIEVAL_EMBEDDING_MODEL_OVERRIDE", ""
+).strip()
 EMBEDDING_MODELS = [
-    "nvidia/llama-nemotron-embed-vl-1b-v2:free",
-    "text-embedding-3-small",  # OpenRouter fallback
+    model
+    for model in [
+        RETRIEVAL_EMBEDDING_MODEL_OVERRIDE or EMBEDDING_MODEL,
+        EMBEDDING_MODEL if RETRIEVAL_EMBEDDING_MODEL_OVERRIDE else "",
+    ]
+    if model
 ]
 
 # ==================== GROQ MODELS ====================
@@ -119,6 +138,17 @@ UPLOAD_RETRY_MAX_ATTEMPTS = int(os.getenv("UPLOAD_RETRY_MAX_ATTEMPTS", "4"))
 UPLOAD_RETRY_BASE_SLEEP_S = float(os.getenv("UPLOAD_RETRY_BASE_SLEEP_S", "2"))
 UPLOAD_RETRY_MAX_SLEEP_S = float(os.getenv("UPLOAD_RETRY_MAX_SLEEP_S", "20"))
 
+# ==================== CELERY / REDIS ====================
+CELERY_VISIBILITY_TIMEOUT_S = int(os.getenv("CELERY_VISIBILITY_TIMEOUT_S", "7200"))
+CELERY_BROKER_HEARTBEAT_S = int(os.getenv("CELERY_BROKER_HEARTBEAT_S", "30"))
+CELERY_BROKER_POOL_LIMIT = int(os.getenv("CELERY_BROKER_POOL_LIMIT", "1"))
+CELERY_REDIS_SOCKET_TIMEOUT_S = float(
+    os.getenv("CELERY_REDIS_SOCKET_TIMEOUT_S", "30")
+)
+CELERY_REDIS_HEALTH_CHECK_INTERVAL_S = int(
+    os.getenv("CELERY_REDIS_HEALTH_CHECK_INTERVAL_S", "30")
+)
+
 # ==================== RETRIEVAL ====================
 CHAT_MEMORY_TURNS = 3
 EMBEDDING_CACHE_SIZE = 256
@@ -127,6 +157,26 @@ RELEVANCE_THRESHOLD = 0.35
 LLM_MAX_TOKENS = 4096
 MAX_CONTEXT_CHARS = 14000
 CATEGORY_SLOTS = 2
+ENABLE_STRICT_OUTPUT_SANITIZER = os.getenv(
+    "ENABLE_STRICT_OUTPUT_SANITIZER", "true"
+).lower() in {"1", "true", "yes"}
+ENABLE_DUPLICATE_CHUNK_COLLAPSE = os.getenv(
+    "ENABLE_DUPLICATE_CHUNK_COLLAPSE", "true"
+).lower() in {"1", "true", "yes"}
+ENABLE_HYDE = os.getenv("ENABLE_HYDE", "false").lower() in {"1", "true", "yes"}
+ENABLE_RETRIEVE_THEN_STUFF = os.getenv(
+    "ENABLE_RETRIEVE_THEN_STUFF", "true"
+).lower() in {"1", "true", "yes"}
+ENABLE_CONTEXTUAL_CHUNKING = os.getenv(
+    "ENABLE_CONTEXTUAL_CHUNKING", "false"
+).lower() in {"1", "true", "yes"}
+FOLLOWUP_SESSION_TTL_S = int(os.getenv("FOLLOWUP_SESSION_TTL_S", "1800"))
+HISTORY_RECENT_TURNS = int(os.getenv("HISTORY_RECENT_TURNS", "3"))
+HISTORY_IMPORTANT_MAX = int(os.getenv("HISTORY_IMPORTANT_MAX", "6"))
+RETRIEVE_THEN_STUFF_K = int(os.getenv("RETRIEVE_THEN_STUFF_K", "12"))
+RETRIEVE_THEN_STUFF_FETCH_K = int(
+    os.getenv("RETRIEVE_THEN_STUFF_FETCH_K", "20")
+)
 
 # ==================== LOGGING ====================
 LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")

backend/core/pipeline_ambiguity.py

@@ -0,0 +1,221 @@
+"""
+Ambiguity / scope safety logic.
+
+Extracted from `backend/core/pipeline.py` to isolate multi-doc clarification
+rules and reduce coupling with retrieval/generation.
+"""
+
+from __future__ import annotations
+
+import logging
+
+log = logging.getLogger("rag_pipeline")
+
+
+def check_query_ambiguity(
+    query: str,
+    access_token: str = None,
+    category: str = None,
+) -> dict:
+    from backend.core import pipeline as pipeline_facade
+
+    AMBIGUITY_GAP = 0.12
+    MIN_MATCH_SCORE = 0.05
+    MIN_WORDS_FOR_SPECIFICITY = 10
+
+    words = query.strip().split()
+    if len(words) > MIN_WORDS_FOR_SPECIFICITY and not pipeline_facade._is_generic_ambiguous_query(query):
+        # Still check if category resolves to a single file — if so, auto-pin it
+        try:
+            supabase = pipeline_facade._build_supabase_client(access_token)
+            user_id = None
+            if access_token:
+                from backend.core.auth_utils import safe_extract_jwt_sub
+
+                user_id = safe_extract_jwt_sub(access_token)
+            files_q = supabase.table("ingested_files").select("file_hash, filename")
+            if user_id:
+                files_q = files_q.eq("user_id", user_id)
+            if category and category != "All":
+                files_q = files_q.eq("document_type", category)
+            files_resp = files_q.execute()
+            files = files_resp.data or []
+            if len(files) == 1:
+                single_hash = files[0]["file_hash"]
+            else:
+                single_hash = None
+
+            if len(files) > 1 and pipeline_facade._query_requires_identity_lookup(query):
+                top_files = sorted(
+                    (
+                        (str(f.get("file_hash") or "").strip(), str(f.get("filename") or "").strip())
+                        for f in files
+                    ),
+                    key=lambda x: (x[1] or x[0]),
+                )
+                top_files = [(h, n) for h, n in top_files if h][:3]
+                options = [
+                    {
+                        "mode": "single",
+                        "label": (name or fhash).replace(".pdf", ""),
+                        "file_hash": fhash,
+                    }
+                    for fhash, name in top_files
+                ]
+                return {
+                    "is_ambiguous": True,
+                    "clarification_question": "Which document do you mean? Please pick one.",
+                    "clarification_options": options,
+                    "top_file_hash": None,
+                }
+        except Exception:
+            single_hash = None
+        return {
+            "is_ambiguous": False,
+            "clarification_question": None,
+            "clarification_options": None,
+            "top_file_hash": single_hash,
+        }
+
+    try:
+        supabase = pipeline_facade._build_supabase_client(access_token)
+        user_id = None
+        if access_token:
+            from backend.core.auth_utils import safe_extract_jwt_sub
+
+            user_id = safe_extract_jwt_sub(access_token)
+
+        files_q = supabase.table("ingested_files").select("file_hash, filename")
+        if user_id:
+            files_q = files_q.eq("user_id", user_id)
+        if category and category != "All":
+            files_q = files_q.eq("document_type", category)
+        files_resp = files_q.execute()
+        files = files_resp.data or []
+        if len(files) == 0:
+            return {
+                "is_ambiguous": False,
+                "clarification_question": None,
+                "clarification_options": None,
+                "top_file_hash": None,
+            }
+        if len(files) == 1:
+            return {
+                "is_ambiguous": False,
+                "clarification_question": None,
+                "clarification_options": None,
+                "top_file_hash": files[0]["file_hash"],
+            }
+
+        if pipeline_facade._query_requires_identity_lookup(query):
+            top_files = sorted(
+                (
+                    (str(f.get("file_hash") or "").strip(), str(f.get("filename") or "").strip())
+                    for f in files
+                ),
+                key=lambda x: (x[1] or x[0]),
+            )
+            top_files = [(h, n) for h, n in top_files if h][:3]
+            options = [
+                {
+                    "mode": "single",
+                    "label": (name or fhash).replace(".pdf", ""),
+                    "file_hash": fhash,
+                }
+                for fhash, name in top_files
+            ]
+            return {
+                "is_ambiguous": True,
+                "clarification_question": "Which document do you mean? Please pick one.",
+                "clarification_options": options,
+                "top_file_hash": None,
+            }
+
+        query_vec = pipeline_facade.get_cached_embedding(query)
+        file_scores: list[tuple[str, str, float]] = []  # (file_hash, label, best_score)
+
+        for f in files:
+            fhash = f.get("file_hash")
+            fname = (f.get("filename") or fhash or "Untitled").strip()
+            if not fhash:
+                continue
+            try:
+                resp = supabase.rpc(
+                    "hybrid_search",
+                    {
+                        "query_text": query,
+                        "query_embedding": query_vec,
+                        "match_count": 1,
+                        "filter": {"file_hash": fhash},
+                        "semantic_weight": 0.7,
+                        "keyword_weight": 0.3,
+                        "p_user_id": user_id,
+                    },
+                ).execute()
+                rows = resp.data or []
+                if rows:
+                    score = float(rows[0].get("combined_score", 0.0))
+                    file_scores.append((fhash, fname, score))
+            except Exception as exc:
+                log.warning("Ambiguity check RPC error for %s: %s", str(fhash)[:8], exc)
+
+        if len(file_scores) < 2:
+            return {
+                "is_ambiguous": False,
+                "clarification_question": None,
+                "clarification_options": None,
+                "top_file_hash": None,
+            }
+
+        file_scores.sort(key=lambda x: x[2], reverse=True)
+        top_hash, top_name, top_score = file_scores[0]
+        second_hash, second_name, second_score = file_scores[1]
+        gap = top_score - second_score
+        generic = pipeline_facade._is_generic_ambiguous_query(query)
+
+        log.info(
+            "Ambiguity check: top=%r (%.3f), 2nd=%r (%.3f), gap=%.3f, generic=%s, category=%r",
+            top_name,
+            top_score,
+            second_name,
+            second_score,
+            gap,
+            generic,
+            category,
+        )
+
+        if gap >= AMBIGUITY_GAP and top_score >= MIN_MATCH_SCORE and not generic:
+            return {
+                "is_ambiguous": False,
+                "clarification_question": None,
+                "clarification_options": None,
+                "top_file_hash": top_hash,
+            }
+
+        options = []
+        for fhash, fname, score in file_scores[:3]:
+            options.append(
+                {
+                    "mode": "single",
+                    "label": (fname or fhash).replace(".pdf", ""),
+                    "file_hash": fhash,
+                    "score": round(float(score), 4),
+                }
+            )
+
+        return {
+            "is_ambiguous": True,
+            "clarification_question": "Which document do you mean? Please pick one.",
+            "clarification_options": options,
+            "top_file_hash": None,
+        }
+
+    except Exception as e:
+        log.warning("Ambiguity detector failed: %s", e)
+        return {
+            "is_ambiguous": False,
+            "clarification_question": None,
+            "clarification_options": None,
+            "top_file_hash": None,
+        }
+

backend/core/pipeline_generation.py

@@ -0,0 +1,54 @@
+"""
+Generation / streaming facade functions.
+
+The implementation lives in `pipeline.py` during migration; this module makes
+generation a distinct unit for debugging and future refactors.
+"""
+
+from __future__ import annotations
+
+from typing import Any, AsyncGenerator, List, Optional, Tuple
+
+from langchain_core.documents import Document
+
+
+def generate_answer(
+    chunks: List[Document],
+    query: str,
+    chat_history: Optional[List[dict]] = None,
+    past_memories: Optional[List[dict]] = None,
+) -> Tuple[str, List[str]]:
+    from backend.core import pipeline as pipeline_facade
+
+    return pipeline_facade._generate_answer_impl(
+        chunks=chunks,
+        query=query,
+        chat_history=chat_history,
+        past_memories=past_memories,
+    )
+
+
+async def generate_answer_stream(
+    chunks: List[Document],
+    query: str,
+    chat_history: Optional[List[dict]] = None,
+    session_id: str = "default_session",
+    access_token: str = None,
+    category: str = None,
+    eval_mode: bool = False,
+    priority_file_hashes: List[str] = None,
+) -> AsyncGenerator[dict, None]:
+    from backend.core import pipeline as pipeline_facade
+
+    async for event in pipeline_facade._generate_answer_stream_impl(
+        chunks=chunks,
+        query=query,
+        chat_history=chat_history,
+        session_id=session_id,
+        access_token=access_token,
+        category=category,
+        eval_mode=eval_mode,
+        priority_file_hashes=priority_file_hashes,
+    ):
+        yield event
+

backend/core/pipeline_ingestion.py

@@ -0,0 +1,465 @@
+"""
+Ingestion entrypoints and helpers.
+
+This module intentionally keeps imports lightweight where possible and defers
+heavy dependencies to function scope. It is part of the gradual de-monolith
+refactor: `backend/core/pipeline.py` remains a stable facade.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import os
+import time
+from types import SimpleNamespace
+from typing import List, Optional
+
+try:
+    import fitz
+except Exception:  # optional at import time (only used for PDF/image helpers)
+    fitz = None
+
+from backend.core.cache_manager import invalidate_user_cache
+from backend.core.pipeline_supabase import _build_service_supabase_client, _build_supabase_client
+
+log = logging.getLogger("rag_pipeline")
+
+
+def get_file_fingerprint(file_path: str) -> str:
+    """SHA-256 hash — collision-resistant dedup key."""
+    hasher = hashlib.sha256()
+    with open(file_path, "rb") as f:
+        for chunk in iter(lambda: f.read(65536), b""):
+            hasher.update(chunk)
+    return hasher.hexdigest()
+
+
+def extract_images_from_pdf(file_path: str) -> dict:
+    """
+    Extract images per page using PyMuPDF.
+    Returns dict: {page_number: [base64_string, ...]}
+    """
+    if fitz is None:
+        log.warning("PyMuPDF (fitz) not installed; skipping image extraction.")
+        return {}
+
+    page_images = {}
+    try:
+        doc = fitz.open(file_path)
+        for page_num in range(len(doc)):
+            page = doc[page_num]
+            images = []
+            for img in page.get_images(full=True):
+                xref = img[0]
+                base_image = doc.extract_image(xref)
+                if base_image and base_image.get("image"):
+                    # --- NEW LOGIC: Junk Image Filter ---
+                    w = base_image.get("width", 0)
+                    h = base_image.get("height", 0)
+
+                    # 1. Skip tiny icons (e.g., smaller than 100x100 pixels)
+                    if w < 100 or h < 100:
+                        continue
+
+                    # 2. Skip extreme aspect ratios (e.g., skinny banners/logos)
+                    aspect_ratio = w / h if h > 0 else 0
+                    if aspect_ratio > 5.0 or aspect_ratio < 0.2:
+                        continue
+                    # ------------------------------------
+                    import base64
+
+                    b64 = base64.b64encode(base_image["image"]).decode("utf-8")
+                    images.append(b64)
+            if images:
+                page_images[page_num + 1] = images  # 1-indexed to match page_numbers
+        doc.close()
+        log.info("PyMuPDF extracted images from %d pages", len(page_images))
+    except Exception as exc:
+        log.warning("PyMuPDF image extraction failed: %s", exc)
+    return page_images
+
+
+def _has_text_layer(pdf_path: str) -> bool:
+    """Check if the PDF has native digital text to skip expensive OCR."""
+    if fitz is None:
+        # Without PyMuPDF we can't cheaply inspect the text layer.
+        return False
+    try:
+        doc = fitz.open(pdf_path)
+        for page in doc:
+            if page.get_text().strip():
+                return True
+        return False
+    except Exception:
+        return False
+
+
+
+def _extract_element_metrics(elements: list) -> dict[str, float]:
+    page_numbers = {
+        getattr(getattr(el, "metadata", None), "page_number", None)
+        for el in elements
+        if getattr(getattr(el, "metadata", None), "page_number", None) is not None
+    }
+    page_count = max(1, len(page_numbers))
+    text_chars = sum(len(el.text) for el in elements if hasattr(el, "text") and el.text)
+    element_count = len(elements)
+    chars_per_page = text_chars / max(1, page_count)
+    return {
+        "text_chars": text_chars,
+        "element_count": element_count,
+        "page_count": page_count,
+        "chars_per_page": chars_per_page,
+    }
+
+
+def _should_retry_with_hi_res(
+    strategy: str,
+    metrics: dict[str, float],
+) -> bool:
+    return (
+        strategy == "fast"
+        and metrics["chars_per_page"] < 200
+        and metrics["element_count"] < 10
+    )
+
+
+def partition_document(file_path: str) -> list:
+    # Dynamic OCR routing + guarded high-resolution retry for suspiciously thin extraction
+    # Use facade symbol so tests can monkeypatch `backend.core.pipeline._has_text_layer`.
+    from backend.core import pipeline as pipeline_facade
+
+    partition_pdf = getattr(pipeline_facade, "partition_pdf", None)
+    if not callable(partition_pdf):
+        try:
+            from unstructured.partition.pdf import partition_pdf as _partition_pdf
+        except Exception as exc:
+            raise RuntimeError(
+                "Missing dependency 'unstructured'. Install it to ingest PDFs."
+            ) from exc
+        partition_pdf = _partition_pdf
+
+    has_text = pipeline_facade._has_text_layer(file_path)
+    strategy = "fast" if has_text else "hi_res"
+    log.info("PDF text layer detected: %s. Using partition strategy: %s", has_text, strategy)
+    elements = partition_pdf(
+        filename=file_path,
+        strategy=strategy,
+        infer_table_structure=True,
+        extract_image_block_types=["Image"],
+        extract_image_block_to_payload=True,
+    )
+    metrics = _extract_element_metrics(elements)
+    log.info(
+        "%d elements extracted (text_chars=%d, page_count=%d, chars_per_page=%.1f)",
+        len(elements),
+        metrics["text_chars"],
+        metrics["page_count"],
+        metrics["chars_per_page"],
+    )
+
+    if _should_retry_with_hi_res(strategy, metrics):
+        log.info(
+            "Extraction looked suspiciously thin (chars_per_page=%.1f, elements=%d) — retrying once with hi_res.",
+            metrics["chars_per_page"],
+            metrics["element_count"],
+        )
+        hi_res_elements = partition_pdf(
+            filename=file_path,
+            strategy="hi_res",
+            infer_table_structure=True,
+            extract_image_block_types=["Image"],
+            extract_image_block_to_payload=True,
+        )
+        hi_res_metrics = _extract_element_metrics(hi_res_elements)
+        if (
+            hi_res_metrics["text_chars"] > metrics["text_chars"]
+            or hi_res_metrics["element_count"] > metrics["element_count"]
+        ):
+            log.info(
+                "Using hi_res extraction instead (text_chars=%d, elements=%d).",
+                hi_res_metrics["text_chars"],
+                hi_res_metrics["element_count"],
+            )
+            return hi_res_elements
+        log.info("Keeping fast extraction — hi_res did not improve coverage.")
+
+    return elements
+
+
+def _build_document_tree(elements: list) -> dict:
+    """
+    Converts a flat list of unstructured elements into a nested JSON tree.
+    Titles become parent nodes, and Text/Tables become their children.
+    """
+    tree = {"title": "Document Root", "type": "root", "children": []}
+    current_section = tree
+
+    for el in elements:
+        category = getattr(el, "category", "Text")
+        text = str(el).strip()
+        if not text:
+            continue
+        page_num = getattr(getattr(el, "metadata", None), "page_number", None)
+        try:
+            page_num = int(page_num) if page_num is not None else None
+        except Exception:
+            page_num = None
+
+        if category == "Title":
+            new_section = {
+                "type": "section",
+                "title": text[:150],  # Keep titles concise
+                "content": text,
+                "children": [],
+            }
+            tree["children"].append(new_section)
+            current_section = new_section
+        elif category in ("Table", "Text", "NarrativeText", "ListItem"):
+            child = {"type": category, "content": text}
+            if page_num is not None:
+                child["page_numbers"] = [page_num]
+            current_section["children"].append(child)
+
+    return tree
+
+
+def run_ingestion(
+    pdf_path: str,
+    export_json: bool = False,
+    force: bool = False,
+    progress_callback=None,
+    original_filename: str = None,
+    access_token: str = None,
+) -> str:
+    """
+    Ingestion orchestrator.
+
+    Note: during the de-monolith refactor, some collaborators still live on the
+    facade module. We import them lazily to avoid circular imports at module load.
+    """
+    from backend.core.auth_utils import extract_jwt_sub
+    from backend.core import pipeline as pipeline_facade
+
+    STEPS = 6
+    stage_timings_ms: dict[str, int] = {}
+
+    def _progress(step: int, msg: str):
+        log.info("[%d/%d] %s", step, STEPS, msg)
+        if progress_callback:
+            progress_callback(step, STEPS, msg)
+
+    def _record_stage_timing(stage_name: str, started_at: float) -> None:
+        elapsed_ms = max(0, int((time.perf_counter() - started_at) * 1000))
+        stage_timings_ms[stage_name] = elapsed_ms
+        log.info("Ingestion stage '%s' completed in %d ms", stage_name, elapsed_ms)
+        pipeline_facade._log_ingestion_retry_event(
+            user_id=user_id,
+            file_hash=file_hash if "file_hash" in locals() else None,
+            batch_num=0,
+            total_batches=0,
+            attempt=1,
+            event_type="stage_timing",
+            message=json.dumps({"stage": stage_name, "elapsed_ms": elapsed_ms})[:500],
+            sleep_s=0,
+        )
+
+    log.info("=" * 50)
+    log.info("Starting ingestion: %s", pdf_path)
+
+    user_id = (
+        extract_jwt_sub(access_token)
+        if access_token
+        else "00000000-0000-0000-0000-000000000000"
+    )
+
+    _progress(1, "Computing file fingerprint…")
+    # Use facade symbol so tests can monkeypatch `backend.core.pipeline.get_file_fingerprint`.
+    file_hash = pipeline_facade.get_file_fingerprint(pdf_path)
+    already_exists = pipeline_facade.is_file_already_ingested(file_hash, access_token=access_token)
+    if not already_exists:
+        recovered_existing = pipeline_facade._recover_or_prepare_orphaned_upload(
+            file_hash,
+            user_id=user_id,
+            access_token=access_token,
+            filename_hint=original_filename or os.path.basename(pdf_path),
+            force=force,
+        )
+        if recovered_existing:
+            return recovered_existing
+    if already_exists and not force:
+        log.info("SKIPPING — already ingested.")
+        return "already_ingested"
+
+    forced_category = None
+    if already_exists or force:
+        try:
+            _sb = pipeline_facade._build_supabase_client(access_token)
+            _existing = (
+                _sb.table("ingested_files")
+                .select("document_type, user_overridden")
+                .eq("user_id", user_id)
+                .eq("file_hash", file_hash)
+                .limit(1)
+                .execute()
+            )
+            if _existing.data and _existing.data[0].get("user_overridden"):
+                forced_category = _existing.data[0]["document_type"]
+                log.info(
+                    "User override active — forcing category '%s', skipping classifier.",
+                    forced_category,
+                )
+        except Exception as _exc:
+            log.warning("Could not check user override: %s", _exc)
+
+    if already_exists or force:
+        pipeline_facade._cleanup_existing_ingestion_fragments(
+            file_hash,
+            user_id=user_id,
+            access_token=access_token,
+        )
+
+    _progress(2, "Partitioning PDF (OCR + layout detection)…")
+    stage_started = time.perf_counter()
+    # Use facade symbols so tests can monkeypatch these helpers.
+    elements = pipeline_facade.partition_document(pdf_path)
+    pdf_images = pipeline_facade.extract_images_from_pdf(pdf_path)
+    if not elements:
+        raise ValueError(
+            "The PDF appears blank or unreadable. "
+            "If scanned, ensure tesseract-ocr is installed."
+        )
+    text_chars = sum(len(el.text) for el in elements if hasattr(el, "text") and el.text)
+    coverage_metrics = _extract_element_metrics(elements)
+    if text_chars < 50:
+        raise ValueError(
+            f"PDF contains almost no readable text ({text_chars} chars). "
+            "May be corrupted or image-only without OCR layer."
+        )
+    identity_json = pipeline_facade._identity_json_from_elements(
+        elements,
+        fallback_title=pipeline_facade._extract_pdf_title(elements, os.path.basename(pdf_path)),
+    )
+    _record_stage_timing("partition", stage_started)
+
+    _progress(3, "Classifying document and building taxonomy…")
+    stage_started = time.perf_counter()
+    graph_data = pipeline_facade.extract_document_entities(
+        elements,
+        access_token=access_token,
+        forced_category=forced_category,
+    )
+    if not graph_data.is_allowed:
+        raise ValueError("Document rejected: appears blank, spam, or unreadable.")
+    log.info("Category: '%s'", graph_data.document_type)
+    _record_stage_timing("classify", stage_started)
+
+    try:
+        log.info("🌳 Generating structural PageIndex tree...")
+        doc_tree = pipeline_facade._build_document_tree(elements)
+        sb = pipeline_facade._build_service_supabase_client()
+        sb.table("document_trees").upsert(
+            {"file_hash": file_hash, "user_id": user_id, "tree_json": doc_tree},
+            on_conflict="user_id,file_hash",
+        ).execute()
+        log.info("✅ PageIndex tree saved to Supabase.")
+    except Exception as e:
+        log.warning("⚠️ Failed to generate/save document tree: %s", e)
+
+    _progress(4, f"Chunking and processing (category: {graph_data.document_type})…")
+    stage_started = time.perf_counter()
+    chunks = pipeline_facade.create_chunks(elements, text_chars=text_chars)
+    pdf_path_for_naming = original_filename if original_filename else pdf_path
+    docs, ids = pipeline_facade.process_chunks(
+        chunks,
+        elements,
+        pdf_path_for_naming,
+        file_hash,
+        graph_data,
+        user_id,
+        pdf_images,
+        coverage_metrics=coverage_metrics,
+    )
+    _record_stage_timing("chunk_process", stage_started)
+
+    _progress(5, "Building hierarchical reasoning tree (RAPTOR)...")
+    stage_started = time.perf_counter()
+    docs, ids = pipeline_facade.build_raptor_tree(docs, ids, user_id)
+    pipeline_facade._persist_graph_foundation(
+        user_id=user_id,
+        file_hash=file_hash,
+        docs=docs,
+        graph_data=graph_data,
+    )
+    _record_stage_timing("raptor", stage_started)
+
+    smart_name = docs[0].metadata["source"] if docs else os.path.basename(pdf_path)
+    if export_json:
+        log.info("💾 Exporting processed chunks to local JSON...")
+        pipeline_facade.export_to_json(docs)
+
+    _progress(6, f"Embedding and uploading {len(docs)} tree nodes…")
+    stage_started = time.perf_counter()
+    pipeline_facade.upload_to_supabase(docs, ids, access_token=access_token)
+    _record_stage_timing("upload", stage_started)
+
+    try:
+        sb = pipeline_facade._build_service_supabase_client()
+        sb.table("ingested_files").upsert(
+            {
+                "user_id": user_id,
+                "file_hash": file_hash,
+                "filename": smart_name,
+                "document_type": graph_data.document_type,
+                "chunk_count": len(docs),
+                "identity_json": identity_json,
+            },
+            on_conflict="user_id,file_hash",
+        ).execute()
+        pipeline_facade._log_ingestion_retry_event(
+            user_id=user_id,
+            file_hash=file_hash,
+            batch_num=0,
+            total_batches=0,
+            attempt=1,
+            event_type="registry_saved",
+            message="Registered ingested file after successful upload.",
+        )
+    except Exception as e:
+        log.error("Failed to register file: %s", e)
+        pipeline_facade._log_ingestion_retry_event(
+            user_id=user_id,
+            file_hash=file_hash,
+            batch_num=0,
+            total_batches=0,
+            attempt=1,
+            event_type="registry_failed",
+            message=str(e),
+        )
+
+    if access_token:
+        try:
+            invalidate_user_cache(user_id, reason="new_document_ingested")
+        except Exception:
+            pass
+
+    log.info("Ingestion complete!")
+    pipeline_facade._log_ingestion_retry_event(
+        user_id=user_id,
+        file_hash=file_hash,
+        batch_num=0,
+        total_batches=0,
+        attempt=1,
+        event_type="ingestion_complete",
+        message="Ingestion completed successfully.",
+    )
+    log.info("Ingestion stage timings (ms): %s", stage_timings_ms)
+    return {
+        "pending_review": True,
+        "document_type": graph_data.document_type,
+        "filename": smart_name,
+        "file_hash": file_hash,
+    }
+

backend/core/pipeline_memory.py

@@ -0,0 +1,23 @@
+"""
+Memory & prefetch facade functions.
+
+The implementation lives in `pipeline.py` during migration; this module gives
+it a clear ownership boundary and makes it easy to feature-flag later.
+"""
+
+from __future__ import annotations
+
+
+def _predict_and_prefetch(
+    original_query: str, answer: str, category: str, session_id: str, access_token: str
+):
+    from backend.core import pipeline as pipeline_facade
+
+    return pipeline_facade._predict_and_prefetch_impl(
+        original_query=original_query,
+        answer=answer,
+        category=category,
+        session_id=session_id,
+        access_token=access_token,
+    )
+

backend/core/pipeline_pageindex.py

@@ -0,0 +1,263 @@
+"""
+PageIndex / structural tree path retrieval.
+
+This module isolates TOC/page lookup heuristics and Supabase `document_trees`
+traversal so issues in structural retrieval don't churn the main retrieval path.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import time
+from typing import List
+
+from langchain_core.documents import Document
+
+
+log = logging.getLogger("rag_pipeline")
+
+
+def _should_use_tree_path(query: str) -> bool:
+    """
+    Zero-latency heuristic to route structured/specific queries to PageIndex
+    instead of the standard vector semantic search.
+    """
+    # 1. Regex match for Course Codes (e.g., DSN4097, CSE2001, ENG101)
+    if re.search(r"\b[A-Z]{2,4}\s?[0-9]{3,4}\b", query, re.IGNORECASE):
+        return True
+
+    q = (query or "").lower()
+
+    # 2. Structured-document intents that benefit from PageIndex.
+    # Keep this conservative: over-triggering PageIndex causes irrelevant “structural” hits.
+    if "table of contents" in q or ("contents" in q and "page" in q):
+        return True
+    trigger_words = {"list", "exactly", "code"}
+    query_words = set(q.split())
+    if query_words.intersection(trigger_words):
+        return True
+
+    return False
+
+
+def tree_search(
+    query: str,
+    access_token: str = None,
+    category: str = None,
+    priority_file_hashes: List[str] = None,
+) -> List[Document]:
+    """
+    Navigates the structural JSON trees in Supabase to answer highly specific
+    'Needle in a Haystack' queries (e.g., course codes, exact table lookups).
+    """
+    log.info("🔍 Executing Tree Search for query: %s", query)
+
+    q = (query or "").strip()
+    q_lower = q.lower()
+
+    def _norm_for_match(s: str) -> str:
+        """
+        Lightweight normalization to make TOC matching robust across OCR/partition quirks:
+        - normalize curly quotes/apostrophes to ASCII
+        - lowercase
+        - collapse whitespace
+        """
+        s = str(s or "")
+        s = (
+            s.replace("’", "'")
+            .replace("‘", "'")
+            .replace("“", '"')
+            .replace("”", '"')
+            .replace("`", "'")
+        )
+        s = s.lower()
+        s = re.sub(r"\s+", " ", s).strip()
+        return s
+
+    # 1. Extract the specific targets from the query (e.g., Course Codes)
+    targets = set(re.findall(r"\b[A-Z]{2,4}\s?[0-9]{3,4}\b", q, re.IGNORECASE))
+
+    # Special-case: Table of contents lookups (“what page is X on?”).
+    # Extract the section title inside quotes if present, otherwise fall back
+    # to a small target set to avoid matching the entire tree.
+    toc_lookup = ("table of contents" in q_lower) or ("contents" in q_lower and "page" in q_lower)
+    toc_target = None
+    if toc_lookup and not targets:
+        m = re.search(r"[\"'“”‘’](.+?)[\"'“”‘’]", q)
+        if m:
+            toc_target = m.group(1).strip()
+            if toc_target:
+                # Add normalized variants so “What’s New” matches "What's New" / "Whats New".
+                norm = _norm_for_match(toc_target)
+                if norm:
+                    targets = {norm, norm.replace("'", "")}
+
+    # Fallback: extract important keywords if no explicit course code is found
+    if not targets:
+        trigger_words = {"table", "contents", "list", "exactly", "code", "section", "capstone", "credits"}
+        stopwords = {
+            "what",
+            "is",
+            "the",
+            "how",
+            "many",
+            "for",
+            "in",
+            "a",
+            "of",
+            "to",
+            "on",
+            "only",
+            "page",
+        }
+        words = {w.strip(".,:;!?()[]{}") for w in q_lower.split()}
+        words = {w for w in words if w}
+        targets = words - trigger_words - stopwords
+
+    if not targets:
+        log.info("No specific targets extracted for tree search.")
+        return []
+
+    try:
+        from backend.core.auth_utils import extract_jwt_sub
+        from backend.core import pipeline as pipeline_facade
+
+        user_id = (
+            extract_jwt_sub(access_token)
+            if access_token
+            else "00000000-0000-0000-0000-000000000000"
+        )
+        # Use facade symbol so tests can monkeypatch `backend.core.pipeline._build_supabase_client`.
+        sb = pipeline_facade._build_supabase_client(access_token)
+
+        # 2. Fetch all structural trees for this user
+        res = (
+            sb.table("document_trees")
+            .select("file_hash, tree_json")
+            .eq("user_id", user_id)
+            .execute()
+        )
+        if not res.data:
+            return []
+
+        allowed_hashes = None
+        if category and category != "All":
+            try:
+                allowed_res = (
+                    sb.table("ingested_files")
+                    .select("file_hash")
+                    .eq("document_type", category)
+                    .execute()
+                )
+                allowed_hashes = {
+                    row.get("file_hash") for row in (allowed_res.data or []) if row.get("file_hash")
+                }
+            except Exception as exc:
+                log.warning("Could not apply tree-search category filter: %s", exc)
+        if priority_file_hashes:
+            pinned_hashes = {h for h in priority_file_hashes if h}
+            if pinned_hashes:
+                allowed_hashes = (
+                    pinned_hashes
+                    if allowed_hashes is None
+                    else allowed_hashes.intersection(pinned_hashes)
+                )
+
+        matched_chunks: list[Document] = []
+
+        # 3. Recursive Tree Traversal
+        def _traverse(node, parent_title="", file_hash=""):
+            title = str(node.get("title", "") or "")
+            content = str(node.get("content", "") or "")
+            node_text = _norm_for_match(title + " " + content)
+
+            # If the node contains our target noun/code, we capture it
+            norm_targets = [_norm_for_match(t) for t in targets]
+            is_match = any(t and t in node_text for t in norm_targets)
+
+            if is_match and content:
+                parent_chain = f"{parent_title} {title}".strip().lower()
+
+                # TOC lookups should only match TOC entries (not random headers/sections that mention the phrase).
+                # NOTE: Many PDFs don't label the TOC as a distinct "Title" element during partitioning,
+                # so TOC rows can end up under "Document Root" or a different parent. We therefore treat
+                # "in TOC section" as a relevance boost (not a hard filter) and rely on the stricter
+                # "dotted leader -> page number" extraction below to keep TOC matches precise.
+                in_toc_section = False
+                if toc_lookup:
+                    in_toc_section = ("table of contents" in parent_chain) or (
+                        parent_chain.startswith("contents") or "contents" in parent_chain
+                    )
+
+                # Score matches: prefer nodes that contain the full target phrase and a TOC-like page number.
+                score = 0.2
+                if toc_lookup and in_toc_section:
+                    score += 0.15
+                if toc_target and _norm_for_match(toc_target) in node_text:
+                    score += 0.6
+                score += 0.2 if any(t and t in node_text for t in norm_targets) else 0.0
+
+                # Attempt to extract page numbers from TOC lines ("..... 6") or "Page 6".
+                page_numbers: list[int] = []
+                # TOC dotted leaders can appear as "..... 6" or ". . . . 6"
+                toc_page_match = re.search(r"(?:\.\s*){2,}(\d{1,3})\b", content)
+                if toc_page_match:
+                    page_numbers.append(int(toc_page_match.group(1)))
+                    score += 0.3
+                elif toc_lookup:
+                    leader_page = re.search(
+                        r"(?:[.\u00b7\u2026]\s*){1,}(\d{1,3})\s*$", content
+                    )
+                    if leader_page:
+                        page_numbers.append(int(leader_page.group(1)))
+                        score += 0.25
+                    else:
+                        spaced_page = re.search(r"\s{2,}(\d{1,3})\s*$", content)
+                        if spaced_page:
+                            page_numbers.append(int(spaced_page.group(1)))
+                            score += 0.2
+                elif not toc_lookup:
+                    page_hint = re.search(
+                        r"\bpage\s+(\d{1,3})\b", content, flags=re.IGNORECASE
+                    )
+                    if page_hint:
+                        page_numbers.append(int(page_hint.group(1)))
+                        score += 0.2
+
+                if toc_lookup and not page_numbers:
+                    return
+
+                matched_chunks.append(
+                    Document(
+                        page_content=f"Section Context: {parent_title} -> {title}\n\n{content}",
+                        metadata={
+                            "source": "PageIndex Tree Structure",
+                            "file_hash": file_hash,
+                            "type": "structural_node",
+                            "page_numbers": page_numbers,
+                            "relevance_score": round(min(1.0, max(0.0, score)), 4),
+                            "retrieved_at_ms": int(time.time() * 1000),
+                        },
+                    )
+                )
+
+            for child in node.get("children", []):
+                _traverse(child, node.get("title", parent_title), file_hash)
+
+        for tree_row in res.data:
+            if allowed_hashes is not None and tree_row.get("file_hash") not in allowed_hashes:
+                continue
+            _traverse(tree_row["tree_json"], file_hash=tree_row["file_hash"])
+
+        log.info("✅ Tree search found %d matching structural nodes.", len(matched_chunks))
+
+        matched_chunks.sort(
+            key=lambda d: float((d.metadata or {}).get("relevance_score") or 0.0), reverse=True
+        )
+        return matched_chunks[:5]
+
+    except Exception as e:
+        log.warning("⚠️ Tree Search failed, falling back to empty chunks: %s", e)
+        return []
+

backend/core/pipeline_retrieval.py

@@ -0,0 +1,83 @@
+"""
+Retrieval facade functions.
+
+During the gradual de-monolith refactor, we keep the heavy implementations in
+`pipeline.py` (renamed to *_impl) and provide stable entrypoints here. This
+lets API/tests import retrieval without pulling generation/ingestion concerns.
+"""
+
+from __future__ import annotations
+
+from typing import List
+
+from langchain_core.documents import Document
+
+
+def generate_sub_queries(original_query: str, *, route_class: str = "factoid") -> List[str]:
+    from backend.core import pipeline as pipeline_facade
+
+    return pipeline_facade._generate_sub_queries_impl(
+        original_query,
+        route_class=route_class,
+    )
+
+
+def retrieve_chunks(
+    query: str,
+    k: int = 3,
+    source_file: str = None,
+    category: str = None,
+    alpha: float = 0.5,
+    session_id: str = "default_session",
+    access_token: str = None,
+    user_id: str = None,
+    original_query: str = None,
+    eval_mode: bool = False,
+    priority_file_hashes: List[str] = None,
+) -> List[Document]:
+    from backend.core import pipeline as pipeline_facade
+
+    return pipeline_facade._retrieve_chunks_impl(
+        query,
+        k=k,
+        source_file=source_file,
+        category=category,
+        alpha=alpha,
+        session_id=session_id,
+        access_token=access_token,
+        user_id=user_id,
+        original_query=original_query,
+        eval_mode=eval_mode,
+        priority_file_hashes=priority_file_hashes,
+    )
+
+
+def retrieve_chunks_routed(
+    query: str,
+    k: int = 3,
+    source_file: str = None,
+    category: str = None,
+    alpha: float = 0.5,
+    session_id: str = "default_session",
+    access_token: str = None,
+    user_id: str = None,
+    original_query: str = None,
+    eval_mode: bool = False,
+    priority_file_hashes: List[str] = None,
+) -> List[Document]:
+    from backend.core import pipeline as pipeline_facade
+
+    return pipeline_facade._retrieve_chunks_routed_impl(
+        query,
+        k=k,
+        source_file=source_file,
+        category=category,
+        alpha=alpha,
+        session_id=session_id,
+        access_token=access_token,
+        user_id=user_id,
+        original_query=original_query,
+        eval_mode=eval_mode,
+        priority_file_hashes=priority_file_hashes,
+    )
+

backend/core/pipeline_routing.py

@@ -0,0 +1,149 @@
+"""
+Routing and expert selection logic.
+
+This module is extracted from `backend/core/pipeline.py` as part of the
+de-monolith refactor. The facade still owns many helpers; we import them
+lazily to avoid circular imports during migration.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import List, Optional
+
+from backend.core.pipeline_types import RouteDecision
+
+log = logging.getLogger("rag_pipeline")
+
+
+def _classify_query_route_decision(
+    query: str,
+    *,
+    session_id: Optional[str] = None,
+    user_id: Optional[str] = None,
+    priority_file_hashes: Optional[List[str]] = None,
+) -> RouteDecision:
+    from backend.core import pipeline as pipeline_facade
+
+    q = (query or "").strip().lower()
+    if not q:
+        return RouteDecision(route_class="factoid", route_reason="empty_query")
+    if q in {"hi", "hello", "hey", "thanks", "thank you"}:
+        return RouteDecision(route_class="no_retrieval", route_reason="greeting")
+    page_scope = pipeline_facade._detect_page_scope(q)
+    exact_field = pipeline_facade._detect_identity_field(q)
+    if page_scope:
+        return RouteDecision(
+            route_class="page_scoped",
+            route_reason=f"page_scope:{page_scope}",
+            preserve_query=True,
+            disable_memory=True,
+            page_scope=page_scope,
+            exact_field=exact_field,
+        )
+    if exact_field or pipeline_facade._is_exact_fact_query(q):
+        return RouteDecision(
+            route_class="exact_fact",
+            route_reason=f"identity_field:{exact_field or 'generic'}",
+            preserve_query=True,
+            disable_memory=True,
+            exact_field=exact_field,
+        )
+    if pipeline_facade._is_follow_up_reference(query, session_id=session_id, user_id=user_id):
+        return RouteDecision(
+            route_class="follow_up",
+            route_reason="session_reference",
+            preserve_query=False,
+            disable_memory=False,
+        )
+    if pipeline_facade._is_compare_like_query(query) or bool(
+        priority_file_hashes and len(priority_file_hashes) > 1
+    ):
+        return RouteDecision(route_class="compare", route_reason="compare_keywords")
+    if pipeline_facade._is_multi_part_query(query):
+        return RouteDecision(route_class="multi_part", route_reason="multi_part_keywords")
+    if pipeline_facade._is_summary_like_query(query):
+        return RouteDecision(route_class="summary", route_reason="summary_keywords")
+    if pipeline_facade._is_relational_query(query):
+        return RouteDecision(route_class="relational", route_reason="relational_keywords")
+
+    llm_decision = pipeline_facade._llm_route_classifier(
+        query,
+        session_id=session_id,
+        user_id=user_id,
+        priority_file_hashes=priority_file_hashes,
+    )
+    if llm_decision and llm_decision.route_class:
+        return llm_decision
+
+    return RouteDecision(route_class="factoid", route_reason="heuristic_default")
+
+
+def _route_query_experts(
+    query: str,
+    *,
+    session_id: Optional[str] = None,
+    user_id: Optional[str] = None,
+    priority_file_hashes: Optional[List[str]] = None,
+) -> dict:
+    from backend.core import pipeline as pipeline_facade
+
+    q = (query or "").strip()
+    q_lower = q.lower()
+    embedding_scores: dict[str, float] = {}
+    try:
+        query_vec = pipeline_facade.get_cached_embedding(q or "general document information")
+        for expert, prototypes in pipeline_facade._ROUTER_PROTOTYPES.items():
+            sims = [
+                pipeline_facade._vector_cosine(
+                    query_vec, pipeline_facade.get_cached_embedding(proto)
+                )
+                for proto in prototypes
+            ]
+            embedding_scores[expert] = max(0.0, sum(sims) / max(1, len(sims)))
+    except Exception as exc:
+        log.debug("Router embedding stage unavailable: %s", exc)
+        embedding_scores = {expert: 0.2 for expert in pipeline_facade._ROUTER_PROTOTYPES}
+
+    feature_scores = {expert: 0.0 for expert in pipeline_facade._ROUTER_PROTOTYPES}
+    if pipeline_facade._is_summary_like_query(q_lower):
+        feature_scores["raptor_summary"] += 0.35
+    if pipeline_facade._is_compare_like_query(q_lower):
+        feature_scores["hybrid_compare"] += 0.45
+        feature_scores["graph_traversal"] += 0.10
+    if any(
+        token in q_lower
+        for token in ("relationship", "connected", "connection", "link", "linked", "why", "cause")
+    ):
+        feature_scores["graph_traversal"] += 0.35
+    if priority_file_hashes and len(priority_file_hashes) > 1:
+        feature_scores["hybrid_compare"] += 0.15
+    if session_id:
+        session_key = pipeline_facade._session_cache_key(session_id, user_id=user_id)
+        if session_key in pipeline_facade._last_chunks and any(
+            token in q_lower for token in ("it", "this", "that", "previous", "above", "earlier")
+        ):
+            feature_scores["episodic_memory"] += 0.35
+    if not priority_file_hashes:
+        feature_scores["dense_chunk"] += 0.10
+
+    combined = {
+        expert: (embedding_scores.get(expert, 0.0) * 0.65)
+        + (feature_scores.get(expert, 0.0) * 0.35)
+        for expert in pipeline_facade._ROUTER_PROTOTYPES
+    }
+    weights = pipeline_facade._normalize_weight_map(combined)
+    ranked = sorted(weights.items(), key=lambda item: item[1], reverse=True)
+    confidence_gap = ranked[0][1] - ranked[1][1] if len(ranked) > 1 else ranked[0][1]
+    if confidence_gap < 0.06 and len(q.split()) >= 4:
+        llm_weights = pipeline_facade._llm_router_fallback(q)
+        if llm_weights:
+            weights = llm_weights
+            ranked = sorted(weights.items(), key=lambda item: item[1], reverse=True)
+            confidence_gap = ranked[0][1] - ranked[1][1] if len(ranked) > 1 else ranked[0][1]
+    return {
+        "expert_weights": weights,
+        "selected_experts": [expert for expert, score in ranked if score >= 0.18][:3],
+        "confidence": round(confidence_gap, 4),
+    }
+

backend/core/pipeline_supabase.py

@@ -0,0 +1,46 @@
+"""
+Supabase client builders and small DB helpers for the RAG pipeline.
+
+Separated so API/worker code can use Supabase utilities without importing the
+entire pipeline (LLMs, unstructured, etc.).
+"""
+
+from __future__ import annotations
+
+try:
+    from supabase.client import create_client
+except Exception:  # optional at import time
+    create_client = None
+
+from backend.core import config
+
+
+def _build_service_supabase_client():
+    """Service-role client (bypasses RLS). Use only for admin/bootstrap paths."""
+    if create_client is None:
+        raise RuntimeError("Missing dependency 'supabase'. Install supabase-py to use DB features.")
+    return create_client(config.SUPABASE_URL, config.SUPABASE_SERVICE_KEY)
+
+
+def _build_user_supabase_client(access_token: str):
+    if create_client is None:
+        raise RuntimeError("Missing dependency 'supabase'. Install supabase-py to use DB features.")
+    if not config.SUPABASE_ANON_KEY:
+        raise RuntimeError(
+            "SUPABASE_ANON_KEY is not set but a tenant access_token was provided."
+        )
+    client = create_client(config.SUPABASE_URL, config.SUPABASE_ANON_KEY)
+    # supabase-py v2: set JWT for RLS via postgrest auth header
+    client.postgrest.auth(access_token)
+    return client
+
+
+def _build_supabase_client(access_token: str = None):
+    """
+    Default to service role for legacy/internal call paths.
+    API routes should pass access_token so RLS is enforced.
+    """
+    if access_token:
+        return _build_user_supabase_client(access_token)
+    return _build_service_supabase_client()
+

backend/core/pipeline_types.py

@@ -0,0 +1,65 @@
+"""
+Shared Pydantic schemas and lightweight types for the RAG pipeline.
+
+Kept in a separate module so API/worker code can import types without pulling
+in the full pipeline runtime (LLM clients, unstructured, etc.).
+"""
+
+from __future__ import annotations
+
+from typing import List, Optional
+
+from pydantic import BaseModel, Field
+
+
+class DocumentGraphMetadata(BaseModel):
+    """
+    Dynamic taxonomy classification.
+    All fields have safe defaults so partial LLM responses never raise.
+    """
+
+    is_allowed: bool = Field(
+        default=True,
+        description=(
+            "True for any real document with meaningful content. "
+            "False ONLY for blank/empty files, pure spam, or completely unreadable content."
+        ),
+    )
+    document_type: str = Field(
+        default="general_document",
+        description=(
+            "A snake_case category label. Choose from the existing list if a good match exists. "
+            "Otherwise invent a concise new label e.g. 'machine_learning_paper', 'legal_contract'."
+        ),
+    )
+    key_entities: List[str] = Field(
+        default_factory=list,
+        description="Names of algorithms, people, organizations, places, or technologies mentioned.",
+    )
+    primary_topics: List[str] = Field(
+        default_factory=list,
+        description="The 2-3 broad themes of the document.",
+    )
+    brief_summary: str = Field(
+        default="No summary available.",
+        description="A one-sentence summary of what this document is about.",
+    )
+    # Absorb extra fields older LLM responses include — prevents Pydantic crash
+    categories: Optional[List[str]] = Field(default=None, exclude=True)
+    audience: Optional[str] = Field(default=None, exclude=True)
+
+
+class QueryVariants(BaseModel):
+    sub_queries: List[str] = Field(
+        description="1-3 highly optimized, distinct search queries broken down from the original prompt."
+    )
+
+
+class RouteDecision(BaseModel):
+    route_class: str = Field(default="factoid")
+    route_reason: str = Field(default="heuristic_default")
+    preserve_query: bool = Field(default=False)
+    disable_memory: bool = Field(default=False)
+    page_scope: Optional[str] = Field(default=None)
+    exact_field: Optional[str] = Field(default=None)
+

backend/core/rate_limit.py

@@ -0,0 +1,39 @@
+from starlette.requests import Request
+
+from backend.core.auth_utils import is_guest_token
+
+try:
+    from slowapi import Limiter, _rate_limit_exceeded_handler
+    from slowapi.errors import RateLimitExceeded
+    from slowapi.util import get_remote_address
+except Exception:  # optional in minimal/test envs
+    Limiter = None
+    RateLimitExceeded = Exception
+    _rate_limit_exceeded_handler = None
+
+    def get_remote_address(request):  # type: ignore
+        return request.client.host if getattr(request, "client", None) else "unknown"
+
+
+def _rate_limit_key(request: Request) -> str:
+    """Use stricter IP limits for guest workspaces, user token limits otherwise."""
+    token = request.headers.get("X-Auth-Token") or request.headers.get("Authorization")
+    if token and token.startswith("Bearer "):
+        token = token.split(" ", 1)[1]
+    if token and not is_guest_token(token):
+        return token
+    return get_remote_address(request)
+
+
+if Limiter is not None:
+    limiter = Limiter(key_func=_rate_limit_key)
+else:
+    class _NoopLimiter:
+        def limit(self, *_args, **_kwargs):
+            def _decorator(fn):
+                return fn
+
+            return _decorator
+
+    limiter = _NoopLimiter()
+

backend/core/tasks.py

@@ -1,14 +1,60 @@
+import logging
 import os
-from celery import Celery
+
+from backend.core import config
 from backend.core.pipeline import run_ingestion
 
+log = logging.getLogger("morpheus.tasks")
+
 # Initialize Celery pointing to your Redis broker
 REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379/0")
 
-celery_app = Celery("morpheus_worker", broker=REDIS_URL, backend=REDIS_URL)
+try:
+    from celery import Celery
+except Exception:
+    Celery = None
+
+if Celery is not None:
+    celery_app = Celery("morpheus_worker", broker=REDIS_URL, backend=REDIS_URL)
+    celery_app.conf.update(
+        task_track_started=True,
+        task_acks_late=True,
+        task_reject_on_worker_lost=True,
+        worker_cancel_long_running_tasks_on_connection_loss=True,
+        broker_connection_retry_on_startup=True,
+        broker_connection_max_retries=None,
+        broker_heartbeat=config.CELERY_BROKER_HEARTBEAT_S,
+        broker_pool_limit=config.CELERY_BROKER_POOL_LIMIT,
+        broker_transport_options={
+            "visibility_timeout": config.CELERY_VISIBILITY_TIMEOUT_S,
+            "socket_keepalive": True,
+            "socket_timeout": config.CELERY_REDIS_SOCKET_TIMEOUT_S,
+            "socket_connect_timeout": config.CELERY_REDIS_SOCKET_TIMEOUT_S,
+            "retry_on_timeout": True,
+            "health_check_interval": config.CELERY_REDIS_HEALTH_CHECK_INTERVAL_S,
+        },
+        result_backend_transport_options={
+            "visibility_timeout": config.CELERY_VISIBILITY_TIMEOUT_S,
+            "retry_policy": {"timeout": config.CELERY_REDIS_SOCKET_TIMEOUT_S},
+            "health_check_interval": config.CELERY_REDIS_HEALTH_CHECK_INTERVAL_S,
+        },
+    )
+else:
+    celery_app = None
 
-@celery_app.task(bind=True)
-def process_pdf_task(self, tmp_path: str, original_filename: str, access_token: str):
+
+def _cleanup_temp_upload(tmp_path: str) -> None:
+    if not tmp_path:
+        return
+    try:
+        os.unlink(tmp_path)
+    except FileNotFoundError:
+        return
+    except OSError as exc:
+        log.warning("Could not remove temp upload %s: %s", tmp_path, exc)
+
+
+def _process_pdf_task_impl(self, tmp_path: str, original_filename: str, access_token: str):
     """
     This runs in a completely separate background process!
     We pass a progress_callback to run_ingestion so it can report its status.
@@ -21,23 +67,18 @@ def process_pdf_task(self, tmp_path: str, original_filename: str, access_token:
         )
 
     try:
-        # Call your existing pipeline
-        result = run_ingestion(
+        return run_ingestion(
             pdf_path=tmp_path,
             original_filename=original_filename,
             progress_callback=update_progress,
             access_token=access_token,
         )
-        
-        # Cleanup the temp file after the heavy ML job is done
-        try: os.unlink(tmp_path)  # noqa: E701
-        except OSError: pass  # noqa: E701
-        
-        return result
-    except Exception as e:
-        try: os.unlink(tmp_path)  # noqa: E701
-        except OSError: pass  # noqa: E701
-        # Reraising the exception tells Celery the task failed
-        raise Exception(str(e))
-    
-    
+    finally:
+        _cleanup_temp_upload(tmp_path)
+
+
+if celery_app is not None:
+    process_pdf_task = celery_app.task(bind=True)(_process_pdf_task_impl)
+else:
+    def process_pdf_task(*_args, **_kwargs):
+        raise RuntimeError("Celery not installed; background ingestion is unavailable.")

backend/core/warmup_classifier.py

@@ -19,7 +19,6 @@ Usage:
 
 import numpy as np
 import logging
-from supabase.client import create_client
 from dotenv import load_dotenv
 from backend.core import config
 
@@ -32,6 +31,11 @@ log = logging.getLogger("warmup")
 
 
 def warmup():
+    try:
+        from supabase.client import create_client
+    except Exception as exc:
+        raise RuntimeError("Missing dependency 'supabase'. Install supabase-py to warm up classifier.") from exc
+
     supabase = create_client(config.SUPABASE_URL, config.SUPABASE_SERVICE_KEY)
 
     # Step 1 — find which categories already have centroids

backend/eval/run_eval.py

@@ -75,6 +75,72 @@ def _load_from_supabase(
     return res.data or []
 
 
+def load_feedback_dataset_candidates(
+    access_token: Optional[str],
+    user_id: Optional[str],
+    *,
+    limit: int = 50,
+) -> List[Dict[str, Any]]:
+    """
+    Promote explicit user feedback into dataset-shaped rows for offline eval curation.
+    These candidates are intentionally separate from `evaluation_datasets` so we can
+    review them before activation.
+    """
+    from backend.core.pipeline import _build_service_supabase_client
+
+    sb = _build_service_supabase_client()
+    feedback_q = (
+        sb.table("answer_feedback")
+        .select("trace_id, helpful, accepted, reason_code, correction_text, promote_to_eval, user_id")
+        .eq("promote_to_eval", True)
+        .limit(limit)
+    )
+    if user_id:
+        feedback_q = feedback_q.eq("user_id", user_id)
+    feedback_rows = feedback_q.execute().data or []
+    trace_ids = [row.get("trace_id") for row in feedback_rows if row.get("trace_id")]
+    if not trace_ids:
+        return []
+
+    trace_rows = (
+        sb.table("query_traces")
+        .select("trace_id, question, doc_diagnostics, failure_modes, answer_preview")
+        .in_("trace_id", trace_ids)
+        .execute()
+        .data
+        or []
+    )
+    trace_map = {row.get("trace_id"): row for row in trace_rows if row.get("trace_id")}
+
+    dataset_rows: List[Dict[str, Any]] = []
+    seen_trace_ids = set()
+    for feedback in feedback_rows:
+        trace_id = feedback.get("trace_id")
+        if trace_id in seen_trace_ids:
+            continue
+        trace = trace_map.get(trace_id, {})
+        question = (trace.get("question") or "").strip()
+        if not question:
+            continue
+        seen_trace_ids.add(trace_id)
+        correction_text = (feedback.get("correction_text") or "").strip()
+        answer_preview = (trace.get("answer_preview") or "").strip()
+        dataset_rows.append(
+            {
+                "question": question,
+                "gold_context_refs": [],
+                "gold_evidence_text": correction_text or answer_preview,
+                "is_answerable": bool(feedback.get("accepted") or feedback.get("helpful")),
+                "trace_id": trace_id,
+                "failure_modes": trace.get("failure_modes") or [],
+                "doc_diagnostics": trace.get("doc_diagnostics") or [],
+                "reason_code": feedback.get("reason_code"),
+                "source": "feedback_trace",
+            }
+        )
+    return dataset_rows
+
+
 def _parse_csv_floats(s: str) -> List[float]:
     return [float(x.strip()) for x in s.split(",") if x.strip()]
 

backend/main.py

@@ -7,19 +7,6 @@ Production:  gunicorn -w 1 -k uvicorn.workers.UvicornWorker backend.main:app --b
 
 import os
 import sys
-from slowapi import Limiter, _rate_limit_exceeded_handler
-from slowapi.util import get_remote_address
-from slowapi.errors import RateLimitExceeded
-from starlette.requests import Request
-
-
-def _rate_limit_key(request: Request) -> str:
-    """Key rate limits by JWT token (per-user), fall back to IP."""
-    token = request.headers.get("X-Auth-Token") or request.headers.get("Authorization")
-    return token or get_remote_address(request)
-
-
-limiter = Limiter(key_func=_rate_limit_key)
 import logging  # noqa: E402
 import subprocess  # noqa: E402
 from contextlib import asynccontextmanager  # noqa: E402
@@ -32,8 +19,17 @@ from dotenv import load_dotenv  # noqa: E402
 
 load_dotenv()
 
+from backend.core.rate_limit import (  # noqa: E402
+    RateLimitExceeded,
+    _rate_limit_exceeded_handler,
+    limiter,
+)
 from backend.api import auth, corpus, ingest, query, admin, frontend_config  # noqa: E402
-from backend.core.intent_classifier import get_intent_classifier_status  # noqa: E402
+try:  # noqa: E402
+    from backend.core.intent_classifier import get_intent_classifier_status
+except Exception:
+    def get_intent_classifier_status():  # type: ignore
+        return {"ok": False, "reason": "intent_classifier_unavailable"}
 
 log = logging.getLogger("morpheus.main")
 
@@ -87,7 +83,8 @@ app = FastAPI(
 
 # ── Rate limiting ─────────────────────────────────────────────────────────────
 app.state.limiter = limiter
-app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
+if _rate_limit_exceeded_handler is not None:
+    app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
 
 _origins = [
     o.strip() for o in os.getenv("ALLOWED_ORIGINS", "*").split(",") if o.strip()

frontend/index.html

@@ -212,24 +212,26 @@
           onclick="submitLogin()"
         >SIGN IN →</button>
 
-        <!-- Admin panel — collapsed by default -->
-        <details style="margin-top:20px; width:100%;">
-          <summary style="
-            font-family:var(--font-mono); font-size:0.58rem;
-            color:var(--muted); cursor:pointer; letter-spacing:0.12em;
-            text-transform:uppercase; list-style:none; text-align:center;
-          ">▸ Admin access</summary>
-          <div style="margin-top:10px; display:flex; flex-direction:column; gap:6px;">
-            <input type="password" id="adminKey" placeholder="Master admin key…" style="width:100%;box-sizing:border-box;"/>
-            <button class="btn-secondary" onclick="submitAdmin()">GET TODAY'S CODE</button>
-            <div id="adminResult" style="font-family:var(--font-mono);font-size:0.7rem;color:var(--phosphor);min-height:14px;text-align:center;"></div>
-            <div id="auth-toggle-panel" style="display:none; margin-top:12px;">
-              <div class="section-label">AUTH GATE</div>
-              <p id="auth-toggle-label" style="font-size:0.72rem;color:var(--muted);margin-bottom:10px;"></p>
-              <button id="auth-toggle-btn" onclick="toggleAuth()" style="width:100%;padding:9px;border-radius:6px;border:1px solid;font-family:var(--font-mono);font-size:0.72rem;cursor:pointer;letter-spacing:0.08em;transition:all 0.15s;">DISABLE AUTH</button>
-            </div>
-          </div>
-        </details>
+        <button
+          id="guestBtn"
+          class="btn-secondary"
+          style="width:100%; letter-spacing:0.1em; margin-top:10px; display:none;"
+          onclick="submitGuest()"
+        >CONTINUE AS GUEST</button>
+        <label
+          id="guestPersistWrap"
+          style="display:none;width:100%;margin-top:10px;font-size:0.68rem;color:var(--muted);line-height:1.45;text-align:left;"
+        >
+          <input type="checkbox" id="guestPersist" style="margin-right:8px;accent-color:var(--phosphor);" />
+          Keep this guest workspace on this device
+        </label>
+        <div
+          id="guestInfo"
+          style="display:none;font-size:0.68rem;color:var(--muted);text-align:center;margin-top:8px;line-height:1.5;"
+        >
+          Guest mode is isolated and rate-limited. By default it expires when the guest session truly ends.
+        </div>
+
       </div>
     </div>
 
@@ -251,8 +253,19 @@
           <button class="nav-btn" id="nav-chat" onclick="switchView('chat')">
             CHAT
           </button>
+          <button
+            class="nav-btn"
+            id="nav-admin"
+            onclick="switchView('admin')"
+            style="display: none"
+          >
+            ADMIN
+          </button>
         </nav>
         <div class="topbar-right">
+          <div class="stat-pill" id="session-mode-pill" style="display:none;">
+            MODE <span class="val" id="session-mode-label">GUEST</span>
+          </div>
           <div class="stat-pill">
             DOCS <span class="val" id="stat-docs">0</span>
           </div>
@@ -263,6 +276,7 @@
             <div class="conn-dot offline" id="conn-dot"></div>
             <span id="conn-label">OFFLINE</span>
           </div>
+          <button onclick="unlockOperatorTools()" style="font-family:var(--font-mono);font-size:0.6rem;letter-spacing:0.1em;padding:4px 10px;border:1px solid var(--muted);border-radius:4px;background:transparent;color:var(--muted);cursor:pointer;transition:color 0.15s,border-color 0.15s;" onmouseover="this.style.color='var(--phosphor)';this.style.borderColor='var(--phosphor)';" onmouseout="this.style.color='var(--muted)';this.style.borderColor='var(--muted)';">OPERATOR</button>
           <button onclick="signOut()" style="font-family:var(--font-mono);font-size:0.6rem;letter-spacing:0.1em;padding:4px 10px;border:1px solid var(--muted);border-radius:4px;background:transparent;color:var(--muted);cursor:pointer;transition:color 0.15s,border-color 0.15s;" onmouseover="this.style.color='var(--phosphor)';this.style.borderColor='var(--phosphor)';" onmouseout="this.style.color='var(--muted)';this.style.borderColor='var(--muted)';">SIGN OUT</button>
         </div>
       </header>
@@ -486,6 +500,43 @@
             </button>
           </div>
         </div>
+
+        <!-- ── ADMIN VIEW ── -->
+        <div class="view" id="view-admin">
+          <div class="view-header">
+            <div class="view-title">ADMIN REVIEW</div>
+            <div class="view-subtitle">Trace triage, feedback, and eval promotion</div>
+          </div>
+          <div class="view-body" style="padding-top: 12px">
+            <div style="display:flex;gap:8px;flex-wrap:wrap;margin-bottom:14px;">
+              <input type="text" id="adminTraceFailure" placeholder="failure mode" style="flex:1;min-width:120px;" />
+              <input type="text" id="adminTraceCategory" placeholder="category" style="flex:1;min-width:120px;" />
+              <select id="adminTraceRoute" style="flex:1;min-width:120px;">
+                <option value="">All routes</option>
+                <option value="default">default</option>
+                <option value="single">single</option>
+                <option value="generic_pinned">generic_pinned</option>
+                <option value="explicit_compare">explicit_compare</option>
+              </select>
+              <button class="btn-secondary" onclick="refreshAdminDashboard()">REFRESH</button>
+            </div>
+            <div id="adminSummary" style="font-size:0.78rem;color:var(--muted);margin-bottom:12px;"></div>
+            <div style="display:grid;gap:14px;">
+              <div>
+                <div class="section-label">Recent Traces</div>
+                <div id="adminTraceList"></div>
+              </div>
+              <div>
+                <div class="section-label">Trace Detail</div>
+                <div id="adminTraceDetail"></div>
+              </div>
+              <div>
+                <div class="section-label">Recent Feedback</div>
+                <div id="adminFeedbackList"></div>
+              </div>
+            </div>
+          </div>
+        </div>
       </aside>
       <!-- Mobile bottom navigation — must be inside #app for grid to work -->
       <div id="mobile-nav">
@@ -525,7 +576,8 @@
     <script src="js/corpus.js"></script>
     <script src="js/inspect.js"></script>
     <script src="js/chat.js?v=3"></script>
-    <script src="js/main.js"></script>
+    <script src="js/admin.js?v=1"></script>
+    <script src="js/main.js?v=1"></script>
     <script>
       function mobileNav(tab) {
         document
@@ -563,4 +615,4 @@
   }
 </script>
   </body>
-</html>
+</html>

frontend/js/admin.js

@@ -0,0 +1,234 @@
+function _adminBadge(text, tone = 'muted') {
+  const color = tone === 'danger' ? '#fb7185' : tone === 'success' ? '#34d399' : '#93c5fd';
+  return `<span style="display:inline-block;padding:2px 8px;border:1px solid ${color};border-radius:999px;font-size:0.72rem;color:${color};margin-right:6px;">${esc(text)}</span>`;
+}
+
+function _adminPages(pages) {
+  if (!Array.isArray(pages) || !pages.length) return 'none';
+  return pages.join(', ');
+}
+
+function _adminSignalBadges(quality) {
+  const badges = [];
+  badges.push(_adminBadge(`route ${quality.route_class || 'factoid'}`));
+  if (quality.route_reason) badges.push(_adminBadge(`reason ${quality.route_reason}`));
+  badges.push(_adminBadge(`identity ${quality.identity_store_hit ? 'hit' : 'miss'}`, quality.identity_store_hit ? 'success' : 'muted'));
+  if (quality.history_injected) badges.push(_adminBadge('history injected', 'danger'));
+  if (quality.memory_injected) badges.push(_adminBadge('memory injected', 'danger'));
+  if (quality.sanitizer_triggered) badges.push(_adminBadge(`sanitized ${Number(quality.sanitized_token_count || 0)}`, 'danger'));
+  if (quality.page_scope_required) badges.push(_adminBadge(`pages ${quality.page_scope_supported ? 'supported' : 'violated'}`, quality.page_scope_supported ? 'success' : 'danger'));
+  return badges.join('');
+}
+
+function _adminRerankAudit(quality) {
+  const deltas = Array.isArray(quality.rerank_deltas) ? quality.rerank_deltas : [];
+  if (!deltas.length) return '<div class="confirm-zone">No rerank audit captured.</div>';
+  return deltas.slice(0, 8).map(delta => `
+    <div style="padding:8px 10px;border:1px solid #243142;border-radius:8px;background:rgba(10,18,32,0.55);margin-top:8px;">
+      <div style="font-weight:600;color:#dbeafe">${esc(delta.chunk_id || delta.source || 'candidate')}</div>
+      <div style="font-size:0.78rem;color:#94a3b8;">
+        pre ${Number(delta.pre_rank ?? -1)} → post ${Number(delta.post_rank ?? -1)} ·
+        branch ${esc(delta.branch || 'unknown')} ·
+        score ${Number(delta.score ?? 0).toFixed(2)} ·
+        pages ${esc(_adminPages(delta.page_numbers || []))}
+      </div>
+    </div>
+  `).join('');
+}
+
+function _renderTraceSummary(trace) {
+  const failures = Array.isArray(trace.failure_modes) ? trace.failure_modes : [];
+  const experts = Array.isArray(trace.selected_experts) ? trace.selected_experts : [];
+  const quality = trace.quality_metrics || {};
+  return `
+    <div style="padding:12px;border:1px solid #22304a;border-radius:10px;background:rgba(7,12,24,0.72);margin-bottom:10px;">
+      <div style="display:flex;justify-content:space-between;gap:12px;align-items:flex-start;">
+        <div>
+          <div style="font-weight:600;color:#e2e8f0;">${esc(trace.question || 'Untitled trace')}</div>
+          <div style="font-size:0.78rem;color:#94a3b8;margin-top:4px;">${esc(trace.trace_id || '')}</div>
+        </div>
+        <div style="font-size:0.76rem;color:#94a3b8;text-align:right;">
+          <div>${esc(trace.route_mode || 'default')} · ${esc(quality.route_class || 'factoid')}</div>
+          <div>${esc(trace.review_state || 'pending')}</div>
+        </div>
+      </div>
+      <div style="margin-top:10px;">${experts.map(exp => _adminBadge(exp)).join('')}</div>
+      <div style="margin-top:8px;">${_adminSignalBadges(quality)}</div>
+      <div style="margin-top:8px;">${failures.length ? failures.map(f => _adminBadge(f, 'danger')).join('') : _adminBadge('no failure flags', 'success')}</div>
+      <div style="font-size:0.78rem;color:#cbd5e1;margin-top:10px;">
+        relevance ${Number(quality.retrieval_relevance_proxy ?? 0).toFixed(2)} ·
+        balance ${Number(quality.document_balance ?? 0).toFixed(2)} ·
+        thin docs ${Number(quality.thin_doc_count ?? 0)} ·
+        pages ${esc(_adminPages(quality.selected_page_numbers || []))}
+      </div>
+      <div style="display:flex;gap:8px;flex-wrap:wrap;margin-top:10px;">
+        <button class="btn-secondary" onclick="selectAdminTrace('${esc(trace.trace_id)}')">OPEN</button>
+        <button class="btn-secondary" onclick="reviewAdminTrace('${esc(trace.trace_id)}','reviewed')">MARK REVIEWED</button>
+        <button class="btn-danger" onclick="reviewAdminTrace('${esc(trace.trace_id)}','rejected')">REJECT</button>
+      </div>
+    </div>
+  `;
+}
+
+function _renderTraceDetail(trace, feedbackRows) {
+  if (!trace) return '<div class="confirm-zone">No trace selected yet.</div>';
+  const quality = trace.quality_metrics || {};
+  const feedbackHtml = (feedbackRows || []).map(row => `
+    <div style="padding:10px;border:1px solid #243142;border-radius:8px;background:rgba(10,18,32,0.55);margin-top:8px;">
+      <div style="font-size:0.76rem;color:#94a3b8;">Feedback #${row.id} · ${esc(row.review_state || 'pending')}</div>
+      <div style="font-size:0.86rem;color:#e2e8f0;margin-top:4px;">
+        helpful=${String(row.helpful)} · accepted=${String(row.accepted)} · reason=${esc(row.reason_code || 'none')}
+      </div>
+      ${row.correction_text ? `<div style="margin-top:6px;color:#cbd5e1;">${esc(row.correction_text)}</div>` : ''}
+      <div style="display:flex;gap:8px;flex-wrap:wrap;margin-top:10px;">
+        <button class="btn-secondary" onclick="reviewAdminFeedback(${row.id},'reviewed')">REVIEW</button>
+        <button class="btn-danger" onclick="reviewAdminFeedback(${row.id},'rejected')">REJECT</button>
+        <button class="btn-primary" onclick="promoteAdminFeedback(${row.id})">PROMOTE TO EVAL</button>
+      </div>
+    </div>
+  `).join('');
+
+  const diagnostics = (trace.doc_diagnostics || []).map(diag => `
+    <div style="padding:8px 10px;border:1px solid #243142;border-radius:8px;background:rgba(10,18,32,0.55);margin-top:8px;">
+      <div style="font-weight:600;color:#dbeafe">${esc(diag.source || diag.file_hash || 'Unknown')}</div>
+      <div style="font-size:0.78rem;color:#94a3b8;">${esc(diag.reason || 'unknown')} · support ${esc(diag.support_label || diag.confidence_label || 'unknown')} · candidates ${Number(diag.candidate_count ?? 0)}</div>
+    </div>
+  `).join('');
+
+  return `
+    <div style="padding:12px;border:1px solid #22304a;border-radius:10px;background:rgba(7,12,24,0.72);">
+      <div style="font-size:0.78rem;color:#7dd3fc;letter-spacing:0.12em;text-transform:uppercase;">Question</div>
+      <div style="color:#e2e8f0;margin-top:6px;">${esc(trace.question || '')}</div>
+      <div style="font-size:0.78rem;color:#94a3b8;margin-top:10px;">${esc(trace.trace_id || '')}</div>
+      <div style="margin-top:12px;">
+        ${(trace.failure_modes || []).map(flag => _adminBadge(flag, 'danger')).join('')}
+      </div>
+      <div style="margin-top:10px;">${_adminSignalBadges(quality)}</div>
+      <div style="margin-top:14px;font-size:0.8rem;color:#7dd3fc;letter-spacing:0.12em;text-transform:uppercase;">Trace Signals</div>
+      <div style="margin-top:8px;padding:10px;border:1px solid #243142;border-radius:8px;background:rgba(10,18,32,0.55);color:#cbd5e1;">
+        <div>route ${esc(quality.route_class || 'factoid')} · ${esc(quality.route_reason || 'heuristic_default')}</div>
+        <div style="margin-top:4px;">identity store ${esc(quality.identity_store_hit ? 'hit' : 'miss')} · history ${esc(quality.history_injected ? 'yes' : 'no')} · memory ${esc(quality.memory_injected ? 'yes' : 'no')}</div>
+        <div style="margin-top:4px;">pages ${esc(_adminPages(quality.selected_page_numbers || []))} · opening candidates ${Number(quality.opening_page_candidate_count ?? 0)} · opening selected ${Number(quality.opening_page_selected_count ?? 0)}</div>
+        <div style="margin-top:4px;">page scope ${esc(quality.page_scope_required ? 'required' : 'not required')} · ${esc(quality.page_scope_supported ? 'supported' : 'violated')}</div>
+        <div style="margin-top:4px;">sanitizer ${esc(quality.sanitizer_triggered ? 'triggered' : 'clean')} · tokens removed ${Number(quality.sanitized_token_count ?? 0)}</div>
+      </div>
+      <div style="margin-top:14px;font-size:0.8rem;color:#7dd3fc;letter-spacing:0.12em;text-transform:uppercase;">Experts</div>
+      <pre style="white-space:pre-wrap;background:rgba(2,6,23,0.9);padding:10px;border-radius:8px;border:1px solid #1e293b;color:#cbd5e1;">${esc(JSON.stringify({
+        selected_experts: trace.selected_experts || [],
+        expert_weights: trace.expert_weights || {},
+        quality_metrics: quality,
+        selected_chunk_ids: trace.selected_chunk_ids || [],
+      }, null, 2))}</pre>
+      <div style="margin-top:14px;font-size:0.8rem;color:#7dd3fc;letter-spacing:0.12em;text-transform:uppercase;">Rerank Audit</div>
+      ${_adminRerankAudit(quality)}
+      <div style="margin-top:14px;font-size:0.8rem;color:#7dd3fc;letter-spacing:0.12em;text-transform:uppercase;">Diagnostics</div>
+      ${diagnostics || '<div class="confirm-zone">No diagnostics captured.</div>'}
+      <div style="margin-top:14px;font-size:0.8rem;color:#7dd3fc;letter-spacing:0.12em;text-transform:uppercase;">Answer Preview</div>
+      <div style="margin-top:8px;padding:10px;border:1px solid #243142;border-radius:8px;background:rgba(10,18,32,0.55);color:#cbd5e1;white-space:pre-wrap;">${esc(trace.answer_preview || '')}</div>
+      <div style="margin-top:14px;font-size:0.8rem;color:#7dd3fc;letter-spacing:0.12em;text-transform:uppercase;">Linked Feedback</div>
+      ${feedbackHtml || '<div class="confirm-zone">No linked feedback yet.</div>'}
+    </div>
+  `;
+}
+
+function _renderFeedbackList(rows) {
+  if (!rows.length) return '<div class="confirm-zone">No feedback captured yet.</div>';
+  return rows.map(row => `
+    <div style="padding:10px;border:1px solid #22304a;border-radius:10px;background:rgba(7,12,24,0.72);margin-bottom:10px;">
+      <div style="font-size:0.76rem;color:#94a3b8;">Feedback #${row.id} · trace ${esc(row.trace_id || '')}</div>
+      <div style="color:#e2e8f0;margin-top:4px;">helpful=${String(row.helpful)} · accepted=${String(row.accepted)} · ${esc(row.reason_code || 'no reason')}</div>
+      ${row.correction_text ? `<div style="margin-top:6px;color:#cbd5e1;">${esc(row.correction_text)}</div>` : ''}
+      <div style="display:flex;gap:8px;flex-wrap:wrap;margin-top:10px;">
+        <button class="btn-secondary" onclick="openAdminFeedbackTrace('${esc(row.trace_id || '')}')">OPEN TRACE</button>
+        <button class="btn-secondary" onclick="reviewAdminFeedback(${row.id},'reviewed')">REVIEW</button>
+        <button class="btn-danger" onclick="reviewAdminFeedback(${row.id},'rejected')">REJECT</button>
+        <button class="btn-primary" onclick="promoteAdminFeedback(${row.id})">PROMOTE</button>
+      </div>
+    </div>
+  `).join('');
+}
+
+async function refreshAdminDashboard() {
+  if (!STATE.adminUnlocked || !STATE.adminKey) return;
+  const params = {
+    limit: 20,
+    failure_mode: document.getElementById('adminTraceFailure')?.value || '',
+    category: document.getElementById('adminTraceCategory')?.value || '',
+    route_mode: document.getElementById('adminTraceRoute')?.value || '',
+  };
+  const [traceRes, feedbackRes] = await Promise.all([
+    apiAdminListTraces(STATE.adminKey, params),
+    apiAdminListFeedback(STATE.adminKey, { limit: 20 }),
+  ]);
+  STATE.adminTraces = traceRes.items || [];
+  STATE.adminFeedback = feedbackRes.items || [];
+  document.getElementById('adminSummary').textContent =
+    `${STATE.adminTraces.length} trace(s), ${STATE.adminFeedback.length} feedback row(s) loaded.`;
+  document.getElementById('adminTraceList').innerHTML = STATE.adminTraces.map(_renderTraceSummary).join('');
+  document.getElementById('adminFeedbackList').innerHTML = _renderFeedbackList(STATE.adminFeedback);
+  if (STATE.selectedTraceId) {
+    await selectAdminTrace(STATE.selectedTraceId);
+  } else {
+    document.getElementById('adminTraceDetail').innerHTML = '<div class="confirm-zone">Select a trace to inspect it.</div>';
+  }
+}
+
+async function selectAdminTrace(traceId) {
+  if (!STATE.adminUnlocked || !STATE.adminKey || !traceId) return;
+  STATE.selectedTraceId = traceId;
+  const detail = await apiAdminGetTrace(STATE.adminKey, traceId);
+  document.getElementById('adminTraceDetail').innerHTML = _renderTraceDetail(detail.trace, detail.feedback || []);
+}
+
+async function openAdminFeedbackTrace(traceId) {
+  if (!traceId) return;
+  await selectAdminTrace(traceId);
+  switchView('admin');
+}
+
+async function reviewAdminTrace(traceId, reviewState) {
+  if (!STATE.adminKey) return;
+  const reviewNotes = window.prompt(`Notes for ${reviewState}?`, '') || null;
+  await apiAdminReviewTrace(STATE.adminKey, traceId, {
+    review_state: reviewState,
+    review_notes: reviewNotes,
+  });
+  toast(`Trace marked ${reviewState}.`, 'success');
+  await refreshAdminDashboard();
+}
+
+async function reviewAdminFeedback(feedbackId, reviewState) {
+  if (!STATE.adminKey) return;
+  const reviewNotes = window.prompt(`Notes for ${reviewState}?`, '') || null;
+  await apiAdminReviewFeedback(STATE.adminKey, feedbackId, {
+    review_state: reviewState,
+    review_notes: reviewNotes,
+  });
+  toast(`Feedback marked ${reviewState}.`, 'success');
+  await refreshAdminDashboard();
+}
+
+async function promoteAdminFeedback(feedbackId) {
+  if (!STATE.adminKey) return;
+  await apiAdminPromoteFeedback(STATE.adminKey, feedbackId);
+  toast('Feedback promoted to evaluation_datasets.', 'success');
+  await refreshAdminDashboard();
+}
+
+function enableAdminReview(adminKey) {
+  STATE.adminKey = adminKey;
+  STATE.adminUnlocked = true;
+  const nav = document.getElementById('nav-admin');
+  if (nav) nav.style.display = '';
+  refreshAdminDashboard().catch(err => {
+    toast(`Admin dashboard failed: ${err.message}`, 'error');
+  });
+}
+
+window.refreshAdminDashboard = refreshAdminDashboard;
+window.selectAdminTrace = selectAdminTrace;
+window.openAdminFeedbackTrace = openAdminFeedbackTrace;
+window.reviewAdminTrace = reviewAdminTrace;
+window.reviewAdminFeedback = reviewAdminFeedback;
+window.promoteAdminFeedback = promoteAdminFeedback;
+window.enableAdminReview = enableAdminReview;

frontend/js/api.js

@@ -18,13 +18,38 @@
  */
 async function getSupabaseToken() {
   try {
-    const { data } = await supabaseClient.auth.getSession();
+    const client = await initSupabase();
+    if (!client?.auth) return null;
+    const { data } = await client.auth.getSession();
     return data.session?.access_token ?? null;
   } catch {
     return null;
   }
 }
 
+async function getSupabaseSession() {
+  try {
+    const client = await initSupabase();
+    if (!client?.auth) return null;
+    const { data } = await client.auth.getSession();
+    return data.session ?? null;
+  } catch {
+    return null;
+  }
+}
+
+async function isGuestSession() {
+  const session = await getSupabaseSession();
+  const appMeta = session?.user?.app_metadata || {};
+  const provider = String(appMeta.provider || '').toLowerCase();
+  return Boolean(
+    session?.user?.is_anonymous ||
+    appMeta.is_anonymous ||
+    provider === 'anonymous' ||
+    (Array.isArray(appMeta.providers) && appMeta.providers.includes('anonymous'))
+  );
+}
+
 // ── Core fetch wrapper ────────────────────────────────────────────────────────
 async function apiFetch(path, opts = {}) {
   // Always pull a fresh token — Supabase auto-refreshes silently.
@@ -41,7 +66,7 @@ async function apiFetch(path, opts = {}) {
 
   if (!res.ok) {
     let detail = `HTTP ${res.status}`;
-    try { detail = (await res.json()).detail || detail; } catch {}
+    try { detail = (await res.json()).detail || detail; } catch { }
     throw new Error(detail);
   }
 
@@ -55,7 +80,7 @@ async function apiVerifyPassword(password) {
   // Token injection is handled by apiFetch — no sessionStorage involved.
   const data = await apiFetch('/api/v1/auth/verify', {
     method: 'POST',
-    body:   JSON.stringify({ password }),
+    body: JSON.stringify({ password }),
   });
   return data;
 }
@@ -63,7 +88,68 @@ async function apiVerifyPassword(password) {
 async function apiVerifyAdmin(key) {
   return apiFetch('/api/v1/auth/admin', {
     method: 'POST',
-    body:   JSON.stringify({ password: key }),
+    body: JSON.stringify({ password: key }),
+  });
+}
+
+async function apiCleanupGuestWorkspace() {
+  return apiFetch('/api/v1/auth/guest-workspace', {
+    method: 'DELETE',
+  });
+}
+
+async function apiAdminFetch(path, adminKey, opts = {}) {
+  if (!adminKey) throw new Error('Admin key required.');
+  return apiFetch(path, {
+    ...opts,
+    headers: {
+      'X-Admin-Key': adminKey,
+      ...(opts.headers || {}),
+    },
+  });
+}
+
+async function apiAdminListTraces(adminKey, params = {}) {
+  const qs = new URLSearchParams();
+  Object.entries(params).forEach(([key, value]) => {
+    if (value !== null && value !== undefined && value !== '') qs.set(key, String(value));
+  });
+  return apiAdminFetch(`/api/v1/admin/traces${qs.toString() ? `?${qs}` : ''}`, adminKey);
+}
+
+async function apiAdminGetTrace(adminKey, traceId) {
+  return apiAdminFetch(`/api/v1/admin/traces/${traceId}`, adminKey);
+}
+
+async function apiAdminReviewTrace(adminKey, traceId, payload) {
+  return apiAdminFetch(`/api/v1/admin/traces/${traceId}/review`, adminKey, {
+    method: 'POST',
+    body: JSON.stringify(payload),
+  });
+}
+
+async function apiAdminListFeedback(adminKey, params = {}) {
+  const qs = new URLSearchParams();
+  Object.entries(params).forEach(([key, value]) => {
+    if (value !== null && value !== undefined && value !== '') qs.set(key, String(value));
+  });
+  return apiAdminFetch(`/api/v1/admin/feedback${qs.toString() ? `?${qs}` : ''}`, adminKey);
+}
+
+async function apiAdminGetFeedback(adminKey, feedbackId) {
+  return apiAdminFetch(`/api/v1/admin/feedback/${feedbackId}`, adminKey);
+}
+
+async function apiAdminReviewFeedback(adminKey, feedbackId, payload) {
+  return apiAdminFetch(`/api/v1/admin/feedback/${feedbackId}/review`, adminKey, {
+    method: 'POST',
+    body: JSON.stringify(payload),
+  });
+}
+
+async function apiAdminPromoteFeedback(adminKey, feedbackId) {
+  return apiAdminFetch(`/api/v1/admin/feedback/${feedbackId}/promote`, adminKey, {
+    method: 'POST',
   });
 }
 
@@ -75,14 +161,14 @@ async function apiLoadFiles() {
 async function apiOverrideCategory(fileHash, newCategory) {
   return apiFetch('/api/v1/corpus/recategorise', {
     method: 'POST',
-    body:   JSON.stringify({ file_hash: fileHash, new_category: newCategory }),
+    body: JSON.stringify({ file_hash: fileHash, new_category: newCategory }),
   });
 }
 
 async function apiRenameDocument(fileHash, newName) {
   return apiFetch('/api/v1/corpus/rename', {
     method: 'POST',
-    body:   JSON.stringify({ file_hash: fileHash, new_name: newName }),
+    body: JSON.stringify({ file_hash: fileHash, new_name: newName }),
   });
 }
 
@@ -90,6 +176,13 @@ async function apiDeleteDocument(fileHash) {
   return apiFetch(`/api/v1/corpus/${fileHash}`, { method: 'DELETE' });
 }
 
+async function apiSubmitAnswerFeedback(payload) {
+  return apiFetch('/api/v1/query/feedback', {
+    method: 'POST',
+    body: JSON.stringify(payload),
+  });
+}
+
 // ── Ingest ────────────────────────────────────────────────────────────────────
 async function apiIngestFile(file) {
   // multipart/form-data — cannot go through apiFetch (no JSON body),
@@ -100,15 +193,15 @@ async function apiIngestFile(file) {
   formData.append('file', file);
 
   const res = await fetch(`${CONFIG.API_URL}/api/v1/ingest/upload`, {
-    method:  'POST',
+    method: 'POST',
     headers: token ? { 'X-Auth-Token': token } : {},
-    body:    formData,
+    body: formData,
   });
 
   if (res.status === 409) throw new Error('already_ingested');
   if (!res.ok) {
     let detail = `HTTP ${res.status}`;
-    try { detail = (await res.json()).detail || detail; } catch {}
+    try { detail = (await res.json()).detail || detail; } catch { }
     throw new Error(detail);
   }
   return res.json();
@@ -119,41 +212,42 @@ async function apiIngestStatus(taskId) {
 }
 
 // ── Query ─────────────────────────────────────────────────────────────────────
-async function apiQuery(query, category, history, sessionId, alpha, callbacks) {
+async function apiQuery(query, category, history, sessionId, alpha, callbacks, pinnedFiles) {
   /**
    * SSE streaming query.
    * callbacks = {
    *   onToken(text)           — called for each streamed token
-   *   onDone(sources, images) — called when stream ends
+   *   onDone({ sources, images, traceId, docDiagnostics }) — called when stream ends
    *   onError(msg)            — called on error
    * }
    */
   const token = await getSupabaseToken();          // ← Supabase JWT
 
   const res = await fetch(`${CONFIG.API_URL}/api/v1/query`, {
-    method:  'POST',
+    method: 'POST',
     headers: {
       'Content-Type': 'application/json',
       ...(token ? { 'X-Auth-Token': token } : {}),
     },
     body: JSON.stringify({
       query,
-      category:   category   || 'All',
-      history:    history    || [],
-      session_id: sessionId  || 'default_session',
-      alpha:      alpha      ?? 0.5,
+      category: category || 'All',
+      history: history || [],
+      session_id: sessionId || 'default_session',
+      alpha: alpha ?? 0.5,
+      priority_file_hashes: pinnedFiles || [],
     }),
   });
 
   if (!res.ok) {
     let detail = `HTTP ${res.status}`;
-    try { detail = (await res.json()).detail || detail; } catch {}
+    try { detail = (await res.json()).detail || detail; } catch { }
     throw new Error(detail);
   }
 
-  const reader  = res.body.getReader();
+  const reader = res.body.getReader();
   const decoder = new TextDecoder();
-  let   buffer  = '';
+  let buffer = '';
 
   while (true) {
     const { done, value } = await reader.read();
@@ -169,10 +263,18 @@ async function apiQuery(query, category, history, sessionId, alpha, callbacks) {
       if (!raw) continue;
       try {
         const event = JSON.parse(raw);
-        if      (event.type === 'token' && callbacks?.onToken)  callbacks.onToken(event.content);
-        else if (event.type === 'done'  && callbacks?.onDone)   callbacks.onDone(event.sources || [], event.images || []);
-        else if (event.type === 'error' && callbacks?.onError)  callbacks.onError(event.content);
-      } catch {}
+        if (event.type === 'token' && callbacks?.onToken) callbacks.onToken(event.content);
+        else if (event.type === 'done' && callbacks?.onDone) {
+          callbacks.onDone({
+            sources: event.sources || [],
+            images: event.images || [],
+            traceId: event.trace_id || null,
+            docDiagnostics: event.doc_diagnostics || [],
+          });
+        }
+        else if (event.type === 'error' && callbacks?.onError) callbacks.onError(event.content);
+        else if (event.type === 'clarification_options' && callbacks?.onOptions) callbacks.onOptions(event.options);
+      } catch { }
     }
   }
-}
+}

frontend/js/chat.js

@@ -10,7 +10,7 @@
   lb.style.cssText = `display:none;position:fixed;inset:0;background:rgba(0,0,0,0.88);
     z-index:9998;align-items:center;justify-content:center;cursor:zoom-out;
     backdrop-filter:blur(4px);`;
-    lb.innerHTML = `
+  lb.innerHTML = `
     <button id="img-lightbox-close"
       onclick="event.stopPropagation(); document.getElementById('img-lightbox').style.display='none'">
       ✕
@@ -34,14 +34,14 @@ function renderMarkdown(text) {
   let inUL = false;
   let inOL = false;
 
-  const closeUL = () => { if (inUL)  { html += '</ul>'; inUL = false; } };
-  const closeOL = () => { if (inOL)  { html += '</ol>'; inOL = false; } };
+  const closeUL = () => { if (inUL) { html += '</ul>'; inUL = false; } };
+  const closeOL = () => { if (inOL) { html += '</ol>'; inOL = false; } };
 
   const inline = (str) => str
     .replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;')
     .replace(/\*\*(.+?)\*\*/g, '<strong>$1</strong>')
-    .replace(/\*(.+?)\*/g,     '<em>$1</em>')
-    .replace(/`([^`]+)`/g,     '<code class="inline-code">$1</code>')
+    .replace(/\*(.+?)\*/g, '<em>$1</em>')
+    .replace(/`([^`]+)`/g, '<code class="inline-code">$1</code>')
     .replace(/\[Source (\d+)\]/g,
       '<span class="source-ref">[S$1]</span>');
 
@@ -100,6 +100,74 @@ function renderMarkdown(text) {
   return html;
 }
 
+function renderDocDiagnostics(docDiagnostics) {
+  if (!Array.isArray(docDiagnostics) || docDiagnostics.length === 0) return '';
+  const rows = docDiagnostics.map(diag => {
+    const score = diag.doc_score != null ? `${Math.round(diag.doc_score * 100)}%` : 'n/a';
+    const reason = diag.reason || 'unknown';
+    const status = diag.included ? 'included' : 'excluded';
+    return `
+      <div style="display:flex;justify-content:space-between;gap:12px;padding:8px 10px;border:1px solid #243142;border-radius:8px;background:rgba(10,18,32,0.55);margin-top:8px;">
+        <div>
+          <div style="font-weight:600;color:#dbeafe">${esc(diag.source || diag.file_hash || 'Unknown')}</div>
+          <div style="font-size:0.85em;color:#94a3b8">${esc(status)} · ${esc(reason)} · candidates ${Number(diag.candidate_count ?? 0)}</div>
+        </div>
+        <div style="font-size:0.85em;color:#cbd5e1;white-space:nowrap">${esc(diag.confidence_label || 'unknown')} · ${esc(score)}</div>
+      </div>
+    `;
+  }).join('');
+  return `
+    <div style="margin-top:12px;padding:12px;border:1px solid #22304a;border-radius:10px;background:rgba(7,12,24,0.72);">
+      <div style="font-size:0.8em;letter-spacing:0.14em;text-transform:uppercase;color:#7dd3fc;">Retrieval Diagnostics</div>
+      ${rows}
+    </div>
+  `;
+}
+
+function attachFeedbackControls(container, traceId) {
+  if (!traceId) return;
+  const bar = document.createElement('div');
+  bar.style.cssText = 'display:flex;flex-wrap:wrap;gap:8px;margin-top:12px;';
+
+  const disableAll = () => {
+    Array.from(bar.querySelectorAll('button')).forEach(btn => { btn.disabled = true; btn.style.opacity = '0.65'; });
+  };
+
+  const makeBtn = (label, handler) => {
+    const btn = document.createElement('button');
+    btn.textContent = label;
+    btn.style.cssText = 'background:rgba(255,255,255,0.05);border:1px solid #334155;color:var(--fg);padding:7px 12px;border-radius:8px;font-size:0.85em;cursor:pointer;';
+    btn.onclick = async () => {
+      try {
+        await handler();
+        disableAll();
+        toast('Feedback saved.', 'success');
+      } catch (err) {
+        toast(err?.message || 'Could not save feedback.', 'error');
+      }
+    };
+    return btn;
+  };
+
+  bar.appendChild(makeBtn('Helpful', async () => {
+    await apiSubmitAnswerFeedback({ trace_id: traceId, helpful: true });
+  }));
+  bar.appendChild(makeBtn('Not Helpful', async () => {
+    const note = window.prompt('What went wrong? You can add a short reason or a correction.', '') || '';
+    await apiSubmitAnswerFeedback({
+      trace_id: traceId,
+      helpful: false,
+      reason_code: note ? 'user_reported_issue' : 'needs_improvement',
+      correction_text: note || null,
+    });
+  }));
+  bar.appendChild(makeBtn('Save Answer', async () => {
+    await apiSubmitAnswerFeedback({ trace_id: traceId, helpful: true, accepted: true });
+  }));
+
+  container.appendChild(bar);
+}
+
 // ── Chat core ─────────────────────────────────────────────
 
 // Debounce guard — prevents double-submit on rapid Enter + button click
@@ -111,7 +179,7 @@ async function sendChat() {
   _lastSendTime = now;
 
   const input = document.getElementById('chatInput');
-  const msg   = input.value.trim();
+  const msg = input.value.trim();
   if (!msg || STATE.isThinking) return;
   input.value = '';
   autoResize(input);
@@ -122,15 +190,15 @@ async function sendChat() {
   document.getElementById('chatSend').disabled = true;
 
   const category = document.getElementById('chatFilterSelect').value;
-  const history  = STATE.chatHistory.slice(-CONFIG.CHAT_HISTORY_TURNS);
+  const history = STATE.chatHistory.slice(-CONFIG.CHAT_HISTORY_TURNS);
 
   // Create assistant bubble immediately — will be filled by stream
   const assistantDiv = appendMsg('assistant', '', [], []);
-  const bubble       = assistantDiv.querySelector('.msg-bubble');
-  bubble.innerHTML   = '<div class="thinking-dots"><span></span><span></span><span></span></div>';
+  const bubble = assistantDiv.querySelector('.msg-bubble');
+  bubble.innerHTML = '<div class="thinking-dots"><span></span><span></span><span></span></div>';
 
-  let   fullText = '';
-  let   started  = false;
+  let fullText = '';
+  let started = false;
 
   try {
     await apiQuery(msg, category, history, STATE.sessionId, STATE.alpha, {
@@ -142,11 +210,11 @@ async function sendChat() {
         fullText += token;
         bubble.innerHTML = renderMarkdown(fullText);
         // Auto scroll
-        document.getElementById('chatMessages').scrollTop = 
-    document.getElementById('chatMessages').scrollHeight;
-    await new Promise(r => setTimeout(r, 0));
+        document.getElementById('chatMessages').scrollTop =
+          document.getElementById('chatMessages').scrollHeight;
+        await new Promise(r => setTimeout(r, 0));
       },
-      onDone(sources, images) {
+      onDone({ sources, images, traceId, docDiagnostics }) {
         // Finalize markdown render
         bubble.innerHTML = renderMarkdown(fullText);
         STATE.chatHistory.push({ role: 'assistant', content: fullText });
@@ -162,11 +230,11 @@ async function sendChat() {
 
         // Append sources
         if (visibleSources.length > 0) {
-          const n     = visibleSources.length;
+          const n = visibleSources.length;
           const chips = visibleSources.map(s => {
-            const score   = s.score != null ? Math.round(s.score * 100) : null;
+            const score = s.score != null ? Math.round(s.score * 100) : null;
             const scoreEl = score != null ? `<span class="source-chip-score">${score}%</span>` : '';
-            const cls     = score == null ? '' : score >= 70 ? '' : score >= 40 ? 'medium' : 'low';
+            const cls = score == null ? '' : score >= 70 ? '' : score >= 40 ? 'medium' : 'low';
             return `<div class="source-chip ${cls}">
               <div class="source-chip-header">
                 <span class="source-chip-name">${esc(s.source)} · chunk ${s.chunk || '?'}</span>
@@ -180,12 +248,18 @@ async function sendChat() {
             <button class="sources-toggle" onclick="
               const p=this.nextElementSibling;
               const open=p.classList.toggle('open');
-              this.textContent=(open?'▲ hide':'▼ show')+' ${n} source${n>1?'s':''}';
+              this.textContent=(open?'▲ hide':'▼ show')+' ${n} source${n > 1 ? 's' : ''}';
             ">▼ show ${n} source${n > 1 ? 's' : ''}</button>
             <div class="sources-panel">${chips}</div>`;
           assistantDiv.appendChild(srcEl);
         }
 
+        if (docDiagnostics && docDiagnostics.length > 0) {
+          const diagEl = document.createElement('div');
+          diagEl.innerHTML = renderDocDiagnostics(docDiagnostics);
+          assistantDiv.appendChild(diagEl);
+        }
+
         // Append images
         if (images.length > 0) {
           const uniqueImages = [...new Set(images)];
@@ -199,6 +273,75 @@ async function sendChat() {
           assistantDiv.appendChild(imgEl);
         }
 
+        attachFeedbackControls(assistantDiv, traceId);
+
+        const el = document.getElementById('chatMessages');
+        el.scrollTop = el.scrollHeight;
+      },
+      onOptions(options) {
+        // Render inline choice buttons
+        const btnContainer = document.createElement('div');
+        btnContainer.style.cssText = 'display:flex;flex-direction:row;flex-wrap:wrap;gap:8px;margin-top:12px;';
+
+        const syncGraphPinStyles = () => {
+          const d3 = window.d3;
+          if (!d3) return;
+
+          d3.selectAll('.node')
+            .filter(d => d && d.type === 'document')
+            .select('circle')
+            .attr('stroke', d => STATE.pinnedFiles.includes(d.file_hash) ? '#ffffff' : d.color)
+            .attr('stroke-width', d => STATE.pinnedFiles.includes(d.file_hash) ? 3 : 1.5)
+            .attr('filter', d => {
+              if (!STATE.pinnedFiles.includes(d.file_hash)) return null;
+              const idx = STATE.categories.indexOf(d.category);
+              return idx >= 0 ? `url(#glow-${idx})` : null;
+            });
+        };
+
+        options.forEach(opt => {
+          const btn = document.createElement('button');
+          btn.textContent = opt.label;
+          btn.style.cssText = `
+            background: rgba(255, 255, 255, 0.05);
+            border: 1px solid #334155;
+            color: var(--fg);
+            padding: 8px 16px;
+            border-radius: 6px;
+            font-size: 0.9em;
+            cursor: pointer;
+            transition: all 0.2s;
+          `;
+          btn.onmouseover = () => {
+            btn.style.background = 'rgba(255, 255, 255, 0.1)';
+            btn.style.borderColor = 'var(--text-glow)';
+          };
+          btn.onmouseout = () => {
+            btn.style.background = 'rgba(255, 255, 255, 0.05)';
+            btn.style.borderColor = '#334155';
+          };
+
+          btn.onclick = () => {
+            // 1) Apply selected routing scope (single-doc or multi-doc)
+            const selectedHashes = opt.mode === 'all'
+              ? (Array.isArray(opt.file_hashes) ? opt.file_hashes.filter(Boolean) : [])
+              : (opt.file_hash ? [opt.file_hash] : []);
+            STATE.pinnedFiles = [...new Set(selectedHashes)];
+            syncGraphPinStyles();
+
+            // 2. Hide the buttons
+            btnContainer.style.display = 'none';
+
+            // 3. Resubmit the query now that it has a pin
+            const input = document.getElementById('chatInput');
+            input.value = msg; // original msg
+            document.getElementById('chatSend').click();
+          };
+          btnContainer.appendChild(btn);
+        });
+
+        assistantDiv.appendChild(btnContainer);
+
         const el = document.getElementById('chatMessages');
         el.scrollTop = el.scrollHeight;
       },
@@ -215,7 +358,7 @@ async function sendChat() {
         }
         bubble.innerHTML = `<p class="msg-p" style="color:var(--red)">${esc(errMsg)}</p>`;
       },
-    });
+    }, STATE.pinnedFiles);
   } catch (e) {
     bubble.innerHTML = `<p class="msg-p" style="color:var(--red)">Request failed: ${esc(e.message)}</p>`;
   } finally {
@@ -225,7 +368,7 @@ async function sendChat() {
 }
 
 function appendMsg(role, text, sources = [], images = []) {
-  const el  = document.getElementById('chatMessages');
+  const el = document.getElementById('chatMessages');
   const div = document.createElement('div');
   div.className = `msg ${role}`;
   const n = sources.length;
@@ -237,11 +380,11 @@ function appendMsg(role, text, sources = [], images = []) {
     imgHtml = `
       <div style="display:flex; flex-direction:row; gap:10px; margin-top:12px; width:100%; overflow-x:auto; padding-bottom:8px;">
         ${uniqueImages.map(img => {
-          const src = img.startsWith('data:') || img.startsWith('http') 
-                      ? img 
-                      : `data:image/jpeg;base64,${img}`;
-          return `<img src="${src}" style="max-height: 220px; max-width: 100%; object-fit: contain; border-radius: 8px; background: white; border: 1px solid #334155; cursor: zoom-in;" onclick="openLightbox(this.src)">`;
-        }).join('')}
+      const src = img.startsWith('data:') || img.startsWith('http')
+        ? img
+        : `data:image/jpeg;base64,${img}`;
+      return `<img src="${src}" style="max-height: 220px; max-width: 100%; object-fit: contain; border-radius: 8px; background: white; border: 1px solid #334155; cursor: zoom-in;" onclick="openLightbox(this.src)">`;
+    }).join('')}
       </div>`;
   }
 
@@ -249,9 +392,9 @@ function appendMsg(role, text, sources = [], images = []) {
   let srcHtml = '';
   if (n > 0) {
     const chips = sources.map(s => {
-      const score   = s.score != null ? Math.round(s.score * 100) : null;
+      const score = s.score != null ? Math.round(s.score * 100) : null;
       const scoreEl = score != null ? `<span class="source-chip-score">${score}%</span>` : '';
-      const cls     = score == null ? '' : score >= 70 ? '' : score >= 40 ? 'medium' : 'low';
+      const cls = score == null ? '' : score >= 70 ? '' : score >= 40 ? 'medium' : 'low';
       return `<div class="source-chip ${cls}">
         <div class="source-chip-header">
           <span class="source-chip-name">${esc(s.source)} · chunk ${s.chunk || '?'}</span>
@@ -264,7 +407,7 @@ function appendMsg(role, text, sources = [], images = []) {
       <button class="sources-toggle" onclick="
         const p=this.nextElementSibling;
         const open=p.classList.toggle('open');
-        this.textContent=(open?'▲ hide':'▼ show')+' ${n} source${n>1?'s':''}';
+        this.textContent=(open?'▲ hide':'▼ show')+' ${n} source${n > 1 ? 's' : ''}';
       ">▼ show ${n} source${n > 1 ? 's' : ''}</button>
       <div class="sources-panel">${chips}</div>`;
   }
@@ -284,7 +427,7 @@ function appendMsg(role, text, sources = [], images = []) {
 }
 
 function appendThinking() {
-  const el  = document.getElementById('chatMessages');
+  const el = document.getElementById('chatMessages');
   const div = document.createElement('div');
   div.className = 'msg assistant';
   div.innerHTML = `

frontend/js/config.js

@@ -2,14 +2,48 @@ const CONFIG = {
   API_URL: '',
   CAT_PALETTE: ['#00ff88','#4a9eff','#f5a623','#ff6b9d','#a78bfa','#34d399','#fb923c','#60a5fa'],
   CHAT_HISTORY_TURNS: 6,
+  GUEST_ENABLED: true,
 };
 
 // Supabase client — keys loaded from backend, never hardcoded here
 let supabaseClient = null;
+let supabaseReady = null;
 
 async function initSupabase() {
-  const res  = await fetch('/api/v1/config');
-  const cfg  = await res.json();
-  const { createClient } = supabase;
-  supabaseClient = createClient(cfg.supabase_url, cfg.supabase_anon);
-}
+  if (supabaseClient?.auth) return supabaseClient;
+  if (supabaseReady) return supabaseReady;
+
+  supabaseReady = (async () => {
+    try {
+      const res = await fetch('/api/v1/config', { cache: 'no-store' });
+      if (!res.ok) {
+        throw new Error(`Config endpoint failed (${res.status})`);
+      }
+
+      const cfg = await res.json();
+      const createClient = window.supabase?.createClient;
+      if (typeof createClient !== 'function') {
+        throw new Error('Supabase browser SDK failed to load.');
+      }
+      if (!cfg?.supabase_url || !cfg?.supabase_anon) {
+        throw new Error('Supabase frontend config is missing.');
+      }
+
+      CONFIG.GUEST_ENABLED = cfg?.guest_enabled !== false;
+      const client = createClient(cfg.supabase_url, cfg.supabase_anon);
+      if (!client?.auth) {
+        throw new Error('Supabase auth client failed to initialize.');
+      }
+
+      supabaseClient = client;
+      window.supabaseClient = client;
+      return client;
+    } catch (err) {
+      supabaseClient = null;
+      supabaseReady = null;
+      throw err;
+    }
+  })();
+
+  return supabaseReady;
+}

frontend/js/corpus.js

@@ -3,6 +3,30 @@
  * Document list, upload (real FastAPI call), category review.
  */
 
+const ACTIVE_INGEST_KEY = 'morpheus_active_ingest';
+let ACTIVE_INGEST_PROMISE = null;
+
+function saveActiveIngest(taskId, filename) {
+  localStorage.setItem(ACTIVE_INGEST_KEY, JSON.stringify({
+    taskId,
+    filename,
+    savedAt: Date.now(),
+  }));
+}
+
+function loadActiveIngest() {
+  try {
+    const raw = localStorage.getItem(ACTIVE_INGEST_KEY);
+    return raw ? JSON.parse(raw) : null;
+  } catch {
+    return null;
+  }
+}
+
+function clearActiveIngest() {
+  localStorage.removeItem(ACTIVE_INGEST_KEY);
+}
+
 // ── Doc list ──────────────────────────────────────────────────────────────────
 function renderDocList() {
   const el = document.getElementById('docList');
@@ -81,6 +105,7 @@ async function processUpload(file) {
   try {
     const queued = await apiIngestFile(file);
     // queued = {task_id, filename, message}
+    saveActiveIngest(queued.task_id, queued.filename || file.name);
 
     setProgress(20, 'Queued — processing in background…');
 
@@ -92,20 +117,71 @@ async function processUpload(file) {
 
     setProgress(100, 'Complete!');
     setTimeout(() => pc.classList.remove('visible'), 1500);
+    clearActiveIngest();
 
-    if (result && result.file_hash) {
+    if (result && result.recovered_existing) {
+      toast('Recovered previous upload without recomputing.', 'success');
+    } else if (result && result.file_hash) {
       showCategoryReview(result.file_hash, result.filename, result.document_type);
     }
     await refreshCorpus();
 
   } catch (err) {
-    pc.classList.remove('visible');
+    if (err.message === 'already_ingested' || err.message === 'Ingestion failed') {
+      clearActiveIngest();
+      pc.classList.remove('visible');
+    }
     if (err.message === 'already_ingested') toast('Already ingested — skipped', 'error');
     else toast('Ingestion failed: ' + err.message, 'error');
   }
   document.getElementById('fileInput').value = '';
 }
 
+async function resumeActiveIngestionIfNeeded() {
+  if (ACTIVE_INGEST_PROMISE) return ACTIVE_INGEST_PROMISE;
+  const active = loadActiveIngest();
+  if (!active || !active.taskId) return null;
+
+  const pc = document.getElementById('progressCard');
+  pc.classList.add('visible');
+  document.getElementById('progressFilename').textContent = active.filename || 'Uploading PDF';
+  setProgress(25, 'Reconnecting to active ingestion…');
+
+  ACTIVE_INGEST_PROMISE = (async () => {
+    try {
+      const result = await pollIngestStatus(active.taskId, (step, total, msg) => {
+        const pct = Math.round((step / total) * 80) + 20;
+        setProgress(pct, msg);
+      });
+
+      clearActiveIngest();
+      setProgress(100, 'Complete!');
+      setTimeout(() => pc.classList.remove('visible'), 1500);
+
+      if (result && result.recovered_existing) {
+        toast('Recovered previous upload without recomputing.', 'success');
+      } else if (result && result.file_hash) {
+        showCategoryReview(result.file_hash, result.filename, result.document_type);
+      }
+      await refreshCorpus();
+      return result;
+    } catch (err) {
+      if (err.message === 'already_ingested' || err.message === 'Ingestion failed') {
+        clearActiveIngest();
+        pc.classList.remove('visible');
+      }
+      if (err.message === 'already_ingested') {
+        await refreshCorpus();
+      }
+      throw err;
+    } finally {
+      ACTIVE_INGEST_PROMISE = null;
+    }
+  })();
+
+  return ACTIVE_INGEST_PROMISE;
+}
+
 async function pollIngestStatus(taskId, onProgress) {
   // No hard timeout — poll until COMPLETED or FAILED.
   // A large PDF with AI vision summaries can take 5-10 minutes on free-tier
@@ -218,4 +294,4 @@ function populateFilterDropdowns() {
   const sel = document.getElementById('chatFilterSelect');
   sel.innerHTML = '<option value="All">All Categories</option>' +
     STATE.categories.map(c => `<option value="${c}">${c.replace(/_/g,' ')}</option>`).join('');
-}
+}

frontend/js/graph.js

@@ -17,10 +17,10 @@
  */
 
 function renderGraph() {
-  const svg   = d3.select('#graph-svg');
+  const svg = d3.select('#graph-svg');
   const panel = document.getElementById('graph-panel');
-  const W     = panel.clientWidth;
-  const H     = panel.clientHeight;
+  const W = panel.clientWidth;
+  const H = panel.clientHeight;
   const empty = document.getElementById('graph-empty');
 
   svg.selectAll('*').remove();
@@ -37,12 +37,12 @@ function renderGraph() {
 
   STATE.categories.forEach(cat => {
     nodes.push({
-      id:    `cat::${cat}`,
-      type:  'category',
+      id: `cat::${cat}`,
+      type: 'category',
       label: cat.replace(/_/g, ' '),
-      raw:   cat,
+      raw: cat,
       color: STATE.catColors[cat],
-      r:     26,
+      r: 26,
       pinned: false,
       count: STATE.files.filter(f => (f.document_type || 'uncategorised') === cat).length,
     });
@@ -51,26 +51,26 @@ function renderGraph() {
   STATE.files.forEach(f => {
     const cat = f.document_type || 'uncategorised';
     nodes.push({
-      id:        `doc::${f.file_hash}`,
-      type:      'document',
-      label:     f.filename,
+      id: `doc::${f.file_hash}`,
+      type: 'document',
+      label: f.filename,
       file_hash: f.file_hash,
-      category:  cat,
-      color:     STATE.catColors[cat] || '#4a9eff',
-      r:         7,
-      pinned:    false,
-      chunks:    f.chunk_count,
-      ingested:  (f.ingested_at || '').slice(0, 10),
+      category: cat,
+      color: STATE.catColors[cat] || '#4a9eff',
+      r: 7,
+      pinned: false,
+      chunks: f.chunk_count,
+      ingested: (f.ingested_at || '').slice(0, 10),
     });
     links.push({ source: `cat::${cat}`, target: `doc::${f.file_hash}` });
   });
 
   // ── Zoom + pan ─────────────────────────────────────────
-  const g    = svg.append('g');
+  const g = svg.append('g');
   const zoom = d3.zoom()
-  .scaleExtent([0.3, 3])
-   // scroll to zoom only, no drag-to-pan
-  .on('zoom', e => g.attr('transform', e.transform));
+    .scaleExtent([0.3, 3])
+    // scroll to zoom only, no drag-to-pan
+    .on('zoom', e => g.attr('transform', e.transform));
   svg.call(zoom).on('dblclick.zoom', null);
   STATE.svgZoom = { zoom, svg };
 
@@ -102,29 +102,56 @@ function renderGraph() {
     .style('cursor', 'pointer')
     .call(d3.drag()
       .on('start', (e, d) => {
-  if (!e.active) STATE.simulation.alphaTarget(0.3).restart();
-  d.fx = d.x; d.fy = d.y;
-  d._lastX = d.x; d._lastY = d.y;
-})
+        if (!e.active) STATE.simulation.alphaTarget(0.3).restart();
+        d.fx = d.x; d.fy = d.y;
+        d._lastX = d.x; d._lastY = d.y;
+      })
       .on('drag', (e, d) => {
-  d._vx = e.x - (d._lastX || e.x);
-  d._vy = e.y - (d._lastY || e.y);
-  d._lastX = e.x; d._lastY = e.y;
-  d.fx = e.x; d.fy = e.y;
-})
+        d._vx = e.x - (d._lastX || e.x);
+        d._vy = e.y - (d._lastY || e.y);
+        d._lastX = e.x; d._lastY = e.y;
+        d.fx = e.x; d.fy = e.y;
+      })
       .on('end', (e, d) => {
-  if (!e.active) STATE.simulation.alphaTarget(0.05);
-  if (!d.pinned) {
-    d.fx = null; d.fy = null;
-    d.vx = (d._vx || 0) * 3;
-    d.vy = (d._vy || 0) * 3;
-    STATE.simulation.alphaTarget(0.3).restart();
-    setTimeout(() => STATE.simulation.alphaTarget(0.05), 2000);
-  }
-})
+        if (!e.active) STATE.simulation.alphaTarget(0.05);
+        if (!d.pinned) {
+          d.fx = null; d.fy = null;
+          d.vx = (d._vx || 0) * 3;
+          d.vy = (d._vy || 0) * 3;
+          STATE.simulation.alphaTarget(0.3).restart();
+          setTimeout(() => STATE.simulation.alphaTarget(0.05), 2000);
+        }
+      })
     )
     .on('click', (event, d) => {
       event.stopPropagation();
+
+      if (d.type === 'document') {
+        // Toggle this document's file_hash in the pinned set
+        const idx = STATE.pinnedFiles.indexOf(d.file_hash);
+        if (idx >= 0) {
+          STATE.pinnedFiles.splice(idx, 1);
+        } else {
+          STATE.pinnedFiles.push(d.file_hash);
+        }
+        // Visual: bright white stroke when pinned, original colour when not
+        node.filter(n => n && n.type === 'document').select('circle')
+          .attr('stroke', n => STATE.pinnedFiles.includes(n.file_hash) ? '#ffffff' : n.color)
+          .attr('stroke-width', n => STATE.pinnedFiles.includes(n.file_hash) ? 3 : 1.5)
+          .attr('filter', n => {
+            if (!STATE.pinnedFiles.includes(n.file_hash)) return null;
+            const glowIdx = STATE.categories.indexOf(n.category);
+            return glowIdx >= 0 ? `url(#glow-${glowIdx})` : null;
+          });
+      } else if (d.type === 'category') {
+        // Clicking a category node clears ALL pins
+        STATE.pinnedFiles = [];
+        node.filter(n => n && n.type === 'document').select('circle')
+          .attr('stroke', n => n.color)
+          .attr('stroke-width', 1.5)
+          .attr('filter', null);
+      }
+
       onNodeClick(d);
     })
     .on('contextmenu', (event, d) => {
@@ -149,7 +176,7 @@ function renderGraph() {
   node.filter(d => d.type === 'category')
     .append('circle')
     .attr('r', 26)
-    .attr('fill',   d => d.color + '18')
+    .attr('fill', d => d.color + '18')
     .attr('stroke', d => d.color)
     .attr('stroke-width', 2)
     .attr('filter', d => {
@@ -170,9 +197,14 @@ function renderGraph() {
   node.filter(d => d.type === 'document')
     .append('circle')
     .attr('r', 7)
-    .attr('fill',   d => d.color + '55')
-    .attr('stroke', d => d.color)
-    .attr('stroke-width', 1.5);
+    .attr('fill', d => d.color + '55')
+    .attr('stroke', d => STATE.pinnedFiles.includes(d.file_hash) ? '#ffffff' : d.color)
+    .attr('stroke-width', d => STATE.pinnedFiles.includes(d.file_hash) ? 3 : 1.5)
+    .attr('filter', d => {
+      if (!STATE.pinnedFiles.includes(d.file_hash)) return null;
+      const glowIdx = STATE.categories.indexOf(d.category);
+      return glowIdx >= 0 ? `url(#glow-${glowIdx})` : null;
+    });
 
   // Labels
   node.append('text')
@@ -180,14 +212,14 @@ function renderGraph() {
     .attr('dy', d => d.type === 'category' ? -32 : -12)
     .attr('text-anchor', 'middle')
     .attr('fill', d => d.type === 'category' ? d.color : 'rgba(200,216,244,0.7)')
-    .attr('font-size',   d => d.type === 'category' ? '10px' : '8px')
+    .attr('font-size', d => d.type === 'category' ? '10px' : '8px')
     .attr('font-family', 'Syne Mono, monospace')
     .attr('font-weight', d => d.type === 'category' ? '600' : '400')
     .text(d => trunc(d.label, d.type === 'category' ? 18 : 16))
     .style('pointer-events', 'none')
     .style('user-select', 'none');
 
-  svg.on('click', () => {});
+  svg.on('click', () => { });
 
   // ── Simulation — Obsidian style ────────────────────────
   STATE.simulation = d3.forceSimulation(nodes)
@@ -207,25 +239,25 @@ function renderGraph() {
     .alphaDecay(0.02)
     .velocityDecay(0.4)
     .on('tick', () => {
-  const liveW = document.getElementById('graph-panel').clientWidth;
-  const liveH = document.getElementById('graph-panel').clientHeight;
-  nodes.forEach(d => {
-  if (d.fx == null) {
-    const pad = 40;
-    if (d.x < pad)         { d.x = pad;          d.vx =  Math.abs(d.vx) * 0.7; }
-    if (d.x > liveW - pad) { d.x = liveW - pad;  d.vx = -Math.abs(d.vx) * 0.7; }
-    if (d.y < pad)         { d.y = pad;           d.vy =  Math.abs(d.vy) * 0.7; }
-    if (d.y > liveH - pad) { d.y = liveH - pad;  d.vy = -Math.abs(d.vy) * 0.7; }
-  }
-});
-  link
-    .attr('x1', d => d.source.x).attr('y1', d => d.source.y)
-    .attr('x2', d => d.target.x).attr('y2', d => d.target.y);
-  node.attr('transform', d => `translate(${d.x},${d.y})`);
-
-  const maxV = Math.max(...nodes.map(d => Math.abs(d.vx||0) + Math.abs(d.vy||0)));
-  if (maxV > 0.5) STATE.simulation.alphaTarget(0.1).restart();
-});
+      const liveW = document.getElementById('graph-panel').clientWidth;
+      const liveH = document.getElementById('graph-panel').clientHeight;
+      nodes.forEach(d => {
+        if (d.fx == null) {
+          const pad = 40;
+          if (d.x < pad) { d.x = pad; d.vx = Math.abs(d.vx) * 0.7; }
+          if (d.x > liveW - pad) { d.x = liveW - pad; d.vx = -Math.abs(d.vx) * 0.7; }
+          if (d.y < pad) { d.y = pad; d.vy = Math.abs(d.vy) * 0.7; }
+          if (d.y > liveH - pad) { d.y = liveH - pad; d.vy = -Math.abs(d.vy) * 0.7; }
+        }
+      });
+      link
+        .attr('x1', d => d.source.x).attr('y1', d => d.source.y)
+        .attr('x2', d => d.target.x).attr('y2', d => d.target.y);
+      node.attr('transform', d => `translate(${d.x},${d.y})`);
+
+      const maxV = Math.max(...nodes.map(d => Math.abs(d.vx || 0) + Math.abs(d.vy || 0)));
+      if (maxV > 0.5) STATE.simulation.alphaTarget(0.1).restart();
+    });
 
   setTimeout(() => STATE.simulation.alphaTarget(0.05), 3000);
 }
@@ -352,7 +384,7 @@ function setupGraphObservers() {
     }
   });
   mo.observe(panel, {
-    attributes:      true,
+    attributes: true,
     attributeFilter: ['style', 'class'],
   });
 
@@ -366,7 +398,7 @@ function setupGraphObservers() {
       if (W && H) graphReheat();
     });
     moParent.observe(panel.parentElement, {
-      attributes:      true,
+      attributes: true,
       attributeFilter: ['style', 'class'],
     });
   }
@@ -377,4 +409,4 @@ function setupGraphObservers() {
   window.addEventListener('resize', () => graphReheat());
 }
 
-setupGraphObservers();
+setupGraphObservers();

frontend/js/main.js

@@ -6,17 +6,126 @@
  *   On success, supabase-js stores the session in localStorage automatically.
  *   getSupabaseToken() in api.js reads it on every request.
  *
- *   The daily-password system is kept ONLY for the admin panel (getting today's
- *   code). It no longer gates the main app — Supabase JWT does that now.
+ *   Legacy daily-password UI has been removed. Supabase JWT gates the main app,
+ *   while the admin key only unlocks operator review tools.
  *
- * Set AUTH_DISABLED = true to skip login during local dev.
+ * AUTH_DISABLED is a local-dev escape hatch only.
+ * Product guest access should use Supabase anonymous sessions instead.
  */
 
-const AUTH_DISABLED = false;  // ← set false in production
+const AUTH_DISABLED = false;  // local dev only — keep false in real use
+const GUEST_PERSIST_KEY = 'morpheus_guest_persist';
+const GUEST_TAB_KEY = 'morpheus_guest_tab_alive';
+const GUEST_LAST_SEEN_KEY = 'morpheus_guest_last_seen_at';
+const GUEST_ACTIVITY_WINDOW_MS = 45000;
+let guestHeartbeatTimer = null;
+
+function shouldPersistGuestWorkspace() {
+  return localStorage.getItem(GUEST_PERSIST_KEY) === '1';
+}
+
+function setGuestPersistPreference(keep) {
+  localStorage.setItem(GUEST_PERSIST_KEY, keep ? '1' : '0');
+  STATE.guestPersist = Boolean(keep);
+}
+
+function markGuestTabAlive() {
+  sessionStorage.setItem(GUEST_TAB_KEY, '1');
+}
+
+function clearGuestSessionMarkers() {
+  sessionStorage.removeItem(GUEST_TAB_KEY);
+  localStorage.removeItem(GUEST_LAST_SEEN_KEY);
+}
+
+function hasGuestTabMarker() {
+  return sessionStorage.getItem(GUEST_TAB_KEY) === '1';
+}
+
+function touchGuestHeartbeat() {
+  localStorage.setItem(GUEST_LAST_SEEN_KEY, String(Date.now()));
+}
+
+function hasRecentGuestHeartbeat() {
+  const raw = Number(localStorage.getItem(GUEST_LAST_SEEN_KEY) || 0);
+  return Number.isFinite(raw) && raw > 0 && (Date.now() - raw) < GUEST_ACTIVITY_WINDOW_MS;
+}
+
+function startGuestHeartbeat() {
+  stopGuestHeartbeat();
+  touchGuestHeartbeat();
+  guestHeartbeatTimer = window.setInterval(() => {
+    if (!STATE.isGuest) return;
+    touchGuestHeartbeat();
+  }, 15000);
+}
+
+function stopGuestHeartbeat() {
+  if (!guestHeartbeatTimer) return;
+  clearInterval(guestHeartbeatTimer);
+  guestHeartbeatTimer = null;
+}
+
+function setGuestControlsVisibility() {
+  const guestBtn = document.getElementById('guestBtn');
+  const guestInfo = document.getElementById('guestInfo');
+  const guestPersistWrap = document.getElementById('guestPersistWrap');
+  const visible = Boolean(CONFIG.GUEST_ENABLED);
+  if (guestBtn) guestBtn.style.display = visible ? '' : 'none';
+  if (guestInfo) guestInfo.style.display = visible ? 'block' : 'none';
+  if (guestPersistWrap) guestPersistWrap.style.display = visible ? 'block' : 'none';
+}
+
+function setSessionMode(session) {
+  const appMeta = session?.user?.app_metadata || {};
+  const provider = String(appMeta.provider || '').toLowerCase();
+  STATE.isGuest = Boolean(
+    session?.user?.is_anonymous ||
+    appMeta.is_anonymous ||
+    provider === 'anonymous' ||
+    (Array.isArray(appMeta.providers) && appMeta.providers.includes('anonymous'))
+  );
+  STATE.guestPersist = STATE.isGuest ? shouldPersistGuestWorkspace() : false;
+
+  const pill = document.getElementById('session-mode-pill');
+  const label = document.getElementById('session-mode-label');
+  if (pill) pill.style.display = STATE.isGuest ? '' : 'none';
+  if (label) label.textContent = STATE.isGuest ? 'GUEST' : 'ACCOUNT';
+}
+
+function isTemporaryGuestResume(session) {
+  if (!session || !STATE.isGuest || STATE.guestPersist) return false;
+  return !hasGuestTabMarker() && !hasRecentGuestHeartbeat();
+}
+
+async function expireTemporaryGuestSession(client) {
+  try {
+    await apiCleanupGuestWorkspace();
+  } catch {
+    // best effort only
+  }
+  try {
+    await client.auth.signOut();
+  } catch {
+    // best effort only
+  }
+  localStorage.removeItem(GUEST_PERSIST_KEY);
+  clearGuestSessionMarkers();
+  STATE.isGuest = false;
+  STATE.guestPersist = false;
+  setSessionMode(null);
+  showLogin();
+  const info = document.getElementById('loginInfo');
+  if (info) {
+    info.textContent = 'Temporary guest workspace expired after the previous guest session ended.';
+    info.style.display = 'block';
+  }
+}
 
 window.addEventListener('DOMContentLoaded', async () => {
   try {
-    await initSupabase();
+    const client = await initSupabase();
+    setGuestControlsVisibility();
 
     if (AUTH_DISABLED) {
       showApp();
@@ -34,27 +143,71 @@ window.addEventListener('DOMContentLoaded', async () => {
     // once with INITIAL_SESSION (with or without a session), then again on
     // SIGNED_IN / SIGNED_OUT. No polling, no timeouts.
     let booted = false;
-    supabaseClient.auth.onAuthStateChange((event, session) => {
-      if (event === 'INITIAL_SESSION') {
-        if (session) {
+    client.auth.onAuthStateChange((event, session) => {
+      const handle = async () => {
+        if (event === 'INITIAL_SESSION') {
+          if (session) {
+            setSessionMode(session);
+            if (isTemporaryGuestResume(session)) {
+              booted = false;
+              await expireTemporaryGuestSession(client);
+              return;
+            }
+            if (STATE.isGuest) {
+              markGuestTabAlive();
+              startGuestHeartbeat();
+            } else {
+              stopGuestHeartbeat();
+            }
+            booted = true;
+            showApp();
+            bootApp();
+          } else {
+            stopGuestHeartbeat();
+            STATE.isGuest = false;
+            STATE.guestPersist = false;
+            showLogin();
+          }
+        } else if (event === 'SIGNED_IN' && !booted) {
+          setSessionMode(session);
+          if (STATE.isGuest) {
+            markGuestTabAlive();
+            startGuestHeartbeat();
+          } else {
+            stopGuestHeartbeat();
+          }
           booted = true;
           showApp();
           bootApp();
-        } else {
+        } else if (event === 'SIGNED_IN') {
+          setSessionMode(session);
+          if (STATE.isGuest) {
+            markGuestTabAlive();
+            startGuestHeartbeat();
+          } else {
+            stopGuestHeartbeat();
+          }
+        } else if (event === 'SIGNED_OUT') {
+          booted = false;
+          stopGuestHeartbeat();
+          STATE.isGuest = false;
+          STATE.guestPersist = false;
+          setSessionMode(null);
           showLogin();
         }
-      } else if (event === 'SIGNED_IN' && !booted) {
-        booted = true;
-        showApp();
-        bootApp();
-      } else if (event === 'SIGNED_OUT') {
-        booted = false;
+      };
+
+      handle().catch(err => {
+        console.error('Auth transition failed:', err);
+        stopGuestHeartbeat();
         showLogin();
-      }
+      });
     });
 
   } catch (err) {
     console.error("Boot failed:", err);
+    const errEl = document.getElementById('loginError');
+    if (errEl) errEl.textContent = 'Auth init failed: ' + err.message;
     showLogin();
   }
 });
@@ -86,7 +239,12 @@ async function submitLogin() {
   err.textContent = '';
 
   try {
-    const {error } = await supabaseClient.auth.signInWithPassword({
+    const client = await initSupabase();
+    if (!client?.auth) {
+      throw new Error('Supabase auth client is unavailable.');
+    }
+
+    const {error } = await client.auth.signInWithPassword({
       email,
       password: pw,
     });
@@ -94,12 +252,14 @@ async function submitLogin() {
     if (error) {
       err.textContent  = error.message || 'Invalid credentials.';
       btn.disabled     = false;
-      btn.textContent  = 'UNLOCK →';
+      btn.textContent  = 'SIGN IN →';
       return;
     }
     // EXPLICIT UI TAKEOVER: 
     // Wait 500ms to guarantee local storage has the token, then force the system online.
     STATE.authenticated = true;
+    const session = await getSupabaseSession();
+    setSessionMode(session);
     showApp();
     
     setTimeout(() => {
@@ -111,7 +271,60 @@ async function submitLogin() {
   } catch (e) {
     err.textContent  = 'Server unreachable: ' + e.message;
     btn.disabled     = false;
-    btn.textContent  = 'UNLOCK →';
+    btn.textContent  = 'SIGN IN →';
+  }
+}
+
+async function submitGuest() {
+  const btn = document.getElementById('guestBtn');
+  const err = document.getElementById('loginError');
+  const info = document.getElementById('loginInfo');
+  const persistCheckbox = document.getElementById('guestPersist');
+  const keepWorkspace = Boolean(persistCheckbox?.checked);
+
+  err.textContent = '';
+  if (info) {
+    info.style.display = 'none';
+    info.textContent = '';
+  }
+
+  btn.disabled = true;
+  btn.textContent = 'STARTING GUEST WORKSPACE…';
+
+  try {
+    const client = await initSupabase();
+    if (!client?.auth) {
+      throw new Error('Supabase auth client is unavailable.');
+    }
+
+    const { error } = await client.auth.signInAnonymously();
+    if (error) {
+      throw error;
+    }
+
+    setGuestPersistPreference(keepWorkspace);
+    const session = await getSupabaseSession();
+    setSessionMode(session);
+    markGuestTabAlive();
+    startGuestHeartbeat();
+    STATE.authenticated = true;
+    showApp();
+    setTimeout(() => {
+      setOnline(true);
+      bootApp();
+      const msg = keepWorkspace
+        ? 'Guest workspace ready. It will stay on this device until you end it.'
+        : 'Temporary guest workspace ready. It will expire after the guest session truly ends.';
+      toast(msg, 'success');
+    }, 300);
+  } catch (e) {
+    err.textContent = e?.message || 'Could not start guest workspace.';
+    if (/anonymous/i.test(err.textContent)) {
+      err.textContent = 'Guest mode is disabled in Supabase Auth settings.';
+    }
+  } finally {
+    btn.disabled = false;
+    btn.textContent = 'CONTINUE AS GUEST';
   }
 }
 
@@ -163,7 +376,12 @@ async function submitSignup() {
   btn.textContent = 'CREATING ACCOUNT…';
 
   try {
-    const { data, error } = await supabaseClient.auth.signUp({ email, password: pw });
+    const client = await initSupabase();
+    if (!client?.auth) {
+      throw new Error('Supabase auth client is unavailable.');
+    }
+
+    const { data, error } = await client.auth.signUp({ email, password: pw });
 
     if (error) {
       err.textContent = error.message || 'Sign-up failed.';
@@ -187,55 +405,43 @@ async function submitSignup() {
   }
 }
 
-// ── Admin panel — daily code (unchanged, still uses master key) ───────────────
-async function submitAdmin() {
-  const key = document.getElementById('adminKey').value.trim();
-  if (!key) return;
+// ── Operator tools unlock ──────────────────────────────────────────────────────
+async function submitAdmin(adminKey) {
+  const key = String(adminKey || '').trim();
+  if (!key) return false;
   try {
     const res = await apiVerifyAdmin(key);
     if (res.valid) {
-      document.getElementById('adminResult').textContent =
-        `Today's code: ${res.token}`;
-      document.getElementById('auth-toggle-panel').style.display = 'block';
-      const locked = localStorage.getItem('nexus_auth_locked') !== 'false';
-      updateToggleUI(locked);
+      if (typeof window.enableAdminReview === 'function') {
+        window.enableAdminReview(key);
+        STATE.adminPendingView = true;
+        if (document.getElementById('app')?.style.display !== 'none') {
+          switchView('admin');
+        } else {
+          const info = document.getElementById('loginInfo');
+          if (info) {
+            info.textContent = 'Admin dashboard unlocked. Sign in to open it.';
+            info.style.display = 'block';
+          }
+        }
+      } else {
+        toast('Admin dashboard assets are stale. Hard refresh with Ctrl+Shift+R.', 'error');
+      }
+      return true;
     } else {
-      document.getElementById('adminResult').textContent = 'Invalid admin key.';
+      toast('Invalid operator key.', 'error');
     }
   } catch (e) {
-    document.getElementById('adminResult').textContent = 'Error: ' + e.message;
+    toast('Operator unlock failed: ' + e.message, 'error');
   }
+  return false;
 }
 
-// ── Auth toggle (admin only) ──────────────────────────────────────────────────
-function updateToggleUI(locked) {
-  const btn   = document.getElementById('auth-toggle-btn');
-  const label = document.getElementById('auth-toggle-label');
-  if (locked) {
-    btn.textContent      = 'DISABLE AUTH';
-    btn.style.background = 'rgba(255,71,87,0.15)';
-    btn.style.borderColor = 'var(--red)';
-    btn.style.color      = 'var(--red)';
-    label.textContent    = 'Auth is ON — users must sign in';
-  } else {
-    btn.textContent      = 'ENABLE AUTH';
-    btn.style.background = 'rgba(0,255,136,0.08)';
-    btn.style.borderColor = 'var(--phosphor)';
-    btn.style.color      = 'var(--phosphor)';
-    label.textContent    = 'Auth is OFF — anyone can access';
-  }
-}
-
-function toggleAuth() {
-  const current = localStorage.getItem('nexus_auth_locked') !== 'false';
-  const next    = !current;
-  localStorage.setItem('nexus_auth_locked', next ? 'true' : 'false');
-  updateToggleUI(next);
-  toast(
-    next ? 'Auth enabled — sign-in required on next visit'
-         : 'Auth disabled — open access',
-    next ? 'error' : 'success',
-  );
+async function unlockOperatorTools() {
+  const key = window.prompt('Enter operator key to open review tools:', '') || '';
+  if (!key.trim()) return;
+  const ok = await submitAdmin(key);
+  if (ok) toast('Operator tools unlocked.', 'success');
 }
 
 function handleLoginKey(e) {
@@ -247,10 +453,38 @@ function handleLoginKey(e) {
 
 // ── Sign out ──────────────────────────────────────────────────────────────────
 async function signOut() {
-  await supabaseClient.auth.signOut();
+  const client = await initSupabase();
+  if (!client?.auth) {
+    throw new Error('Supabase auth client is unavailable.');
+  }
+  if (STATE.isGuest) {
+    if (STATE.guestPersist) {
+      const shouldEnd = window.confirm(
+        'This guest workspace is set to stay on this device. Click OK to end and delete it now, or Cancel to keep it and just close the tab later.'
+      );
+      if (!shouldEnd) return;
+    }
+    try {
+      await apiCleanupGuestWorkspace();
+    } catch (err) {
+      toast('Guest workspace cleanup failed: ' + err.message, 'error');
+    }
+  }
+  await client.auth.signOut();
   STATE.authenticated = false;
+  STATE.isGuest = false;
+  STATE.guestPersist = false;
+  stopGuestHeartbeat();
+  clearGuestSessionMarkers();
+  localStorage.removeItem(GUEST_PERSIST_KEY);
+  setSessionMode(null);
   STATE.files = [];
   STATE.categories = [];
+  STATE.adminUnlocked = false;
+  STATE.adminKey = '';
+  STATE.adminPendingView = false;
+  const navAdmin = document.getElementById('nav-admin');
+  if (navAdmin) navAdmin.style.display = 'none';
   showLogin();
   authTab('signin');
 }
@@ -262,7 +496,22 @@ async function bootApp() {
   setOnline(true);
   try {
     await refreshCorpus();
-    switchView('corpus');
+    if (typeof resumeActiveIngestionIfNeeded === 'function') {
+      resumeActiveIngestionIfNeeded().catch(err => {
+        console.warn('Ingestion resume failed:', err?.message || err);
+      });
+    }
+    if (STATE.adminUnlocked && STATE.adminPendingView) {
+      switchView('admin');
+      STATE.adminPendingView = false;
+      if (typeof refreshAdminDashboard === 'function') {
+        refreshAdminDashboard().catch(err => {
+          toast('Admin dashboard failed: ' + err.message, 'error');
+        });
+      }
+    } else {
+      switchView('corpus');
+    }
   } catch (e) {
     setOnline(false);
     toast('Could not reach backend: ' + e.message, 'error');
@@ -292,4 +541,4 @@ async function refreshCorpus() {
       }
     };
   }, 50);
-})();
+})();

frontend/js/state.js

@@ -3,19 +3,28 @@
  * Single source of truth. All data flows through api.js, never direct Supabase.
  */
 const STATE = {
-  authenticated:   false,
-  files:           [],
-  categories:      [],
-  catColors:       {},
-  simulation:      null,
-  svgZoom:         null,
-  selectedNode:    null,
+  authenticated: false,
+  files: [],
+  categories: [],
+  catColors: {},
+  simulation: null,
+  svgZoom: null,
+  selectedNode: null,
   deleteConfirmed: false,
-  pendingReview:   null,
-  chatHistory:     [],
-  isThinking:      false,
-  sessionId:       crypto.randomUUID(),
+  pendingReview: null,
+  chatHistory: [],
+  isThinking: false,
+  sessionId: crypto.randomUUID(),
   alpha: 0.5,
+  pinnedFiles: [],  // file_hashes of graph-pinned documents
+  adminKey: '',
+  adminUnlocked: false,
+  adminTraces: [],
+  adminFeedback: [],
+  selectedTraceId: null,
+  adminPendingView: false,
+  isGuest: false,
+  guestPersist: false,
 };
 
 function stateRefreshCategories() {
@@ -29,7 +38,7 @@ function stateRefreshCategories() {
 }
 
 async function stateLoadCorpus() {
-  const data  = await apiLoadFiles();
+  const data = await apiLoadFiles();
   STATE.files = data.files || [];
   stateRefreshCategories();
   document.getElementById('stat-docs').textContent = STATE.files.length;

requirements.txt

@@ -22,4 +22,5 @@ celery[redis]
 scikit-learn
 joblib
 sentence-transformers
-python-magic
+python-magic
+pytest

scripts/rebuild_pageindex.py

@@ -0,0 +1,83 @@
+"""
+Rebuild the PageIndex (document_trees) for an already-ingested PDF.
+
+Why this exists:
+- Ingestion deletes the uploaded temp PDF after processing.
+- PageIndex behavior evolves (better TOC handling, page_numbers, etc.).
+- You may want to refresh only the structural index without re-embedding/re-uploading chunks.
+
+Usage (PowerShell):
+  conda activate rag_env
+  python scripts/rebuild_pageindex.py --pdf "C:\path\to\file.pdf" --access-token "<JWT>"
+
+Notes:
+- This only rewrites `document_trees` (and optionally `identity_json` if you choose to extend it).
+- It does NOT touch the vector store, RAPTOR summaries, or ingested_files registry.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+# Ensure repo root is on sys.path so `import backend...` works when executed as a script.
+REPO_ROOT = Path(__file__).resolve().parents[1]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from backend.core.pipeline import (
+    _build_document_tree,
+    _build_service_supabase_client,
+    get_file_fingerprint,
+    partition_document,
+)
+from backend.core.auth_utils import extract_jwt_sub
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Rebuild PageIndex tree for a PDF.")
+    parser.add_argument("--pdf", required=True, help="Path to local PDF file.")
+    parser.add_argument(
+        "--access-token",
+        required=False,
+        default=None,
+        help="User JWT (same X-Auth-Token used by the API). Optional if --user-id is provided.",
+    )
+    parser.add_argument(
+        "--user-id",
+        required=False,
+        default=None,
+        help="Supabase auth user_id (sub). Use this if you don't want to paste a JWT.",
+    )
+    args = parser.parse_args()
+
+    pdf_path = os.path.abspath(args.pdf)
+    if not os.path.exists(pdf_path):
+        raise SystemExit(f"PDF not found: {pdf_path}")
+
+    if args.user_id:
+        user_id = str(args.user_id).strip()
+    elif args.access_token:
+        user_id = extract_jwt_sub(args.access_token)
+    else:
+        raise SystemExit("Provide either --user-id or --access-token.")
+    file_hash = get_file_fingerprint(pdf_path)
+
+    elements = partition_document(pdf_path)
+    doc_tree = _build_document_tree(elements)
+
+    sb = _build_service_supabase_client()
+    sb.table("document_trees").upsert(
+        {"file_hash": file_hash, "user_id": user_id, "tree_json": doc_tree},
+        on_conflict="user_id,file_hash",
+    ).execute()
+
+    print(f"Rebuilt PageIndex tree for file_hash={file_hash} user_id={user_id}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
+

shared/types.py

@@ -1,4 +1,4 @@
-from typing import List, Optional
+from typing import Any, Dict, List, Optional
 from pydantic import BaseModel, Field
 
 class IngestResponse(BaseModel):
@@ -24,16 +24,53 @@ class ChatMessage(BaseModel):
     role: str; content: str
 
 class QueryRequest(BaseModel):
-    query: str; category: str = "All"
-    history: List[ChatMessage] = Field(default_factory=list); k: int = 3
+    query: str
+    category: str = "All"
+    history: List[ChatMessage] = Field(default_factory=list)
+    k: int = 3
     session_id: str = "default_session"
-    alpha: float = 0.5 
+    alpha: float = 0.5
+    priority_file_hashes: List[str] = Field(default_factory=list)
 
 class SourceChunk(BaseModel):
     source: str; score: Optional[float]=None;    chunk: Optional[int | str] = None
     snippet: Optional[str]=None; doc_type: Optional[str]=None
     pages: Optional[List[int]]=None
 
+class DocDiagnostic(BaseModel):
+    file_hash: str
+    source: str
+    included: bool = True
+    candidate_count: int = 0
+    doc_score: Optional[float] = None
+    confidence_label: Optional[str] = None
+    reason: Optional[str] = None
+    support_label: Optional[str] = None
+    thin_doc: Optional[bool] = None
+
+class QueryTrace(BaseModel):
+    trace_id: str
+    query: str
+    session_id: str
+    route_mode: str
+    selected_experts: List[str] = Field(default_factory=list)
+    expert_weights: Dict[str, float] = Field(default_factory=dict)
+    pinned_file_hashes: List[str] = Field(default_factory=list)
+    candidate_counts: Dict[str, int] = Field(default_factory=dict)
+    selected_chunk_ids: List[str] = Field(default_factory=list)
+    doc_diagnostics: List[DocDiagnostic] = Field(default_factory=list)
+    failure_modes: List[str] = Field(default_factory=list)
+    quality_metrics: Dict[str, Any] = Field(default_factory=dict)
+    latency_ms: Optional[int] = None
+    answer_hash: Optional[str] = None
+
+class AnswerFeedback(BaseModel):
+    trace_id: str
+    helpful: Optional[bool] = None
+    accepted: Optional[bool] = None
+    reason_code: Optional[str] = None
+    correction_text: Optional[str] = None
+
 class QueryResponse(BaseModel):
     answer: str; sources: List[SourceChunk] = Field(default_factory=list)
     images: List[str] = []

supabase/migrations/0010_query_traces_feedback_graph.sql

@@ -0,0 +1,131 @@
+create table if not exists public.query_traces (
+    trace_id uuid primary key default gen_random_uuid(),
+    user_id uuid null,
+    session_id text not null default 'default_session',
+    question text not null,
+    route_mode text not null default 'default',
+    selected_experts jsonb not null default '[]'::jsonb,
+    expert_weights jsonb not null default '{}'::jsonb,
+    pinned_file_hashes jsonb not null default '[]'::jsonb,
+    candidate_counts jsonb not null default '{}'::jsonb,
+    selected_chunk_ids jsonb not null default '[]'::jsonb,
+    doc_diagnostics jsonb not null default '[]'::jsonb,
+    failure_modes jsonb not null default '[]'::jsonb,
+    quality_metrics jsonb not null default '{}'::jsonb,
+    answer_hash text null,
+    answer_preview text null,
+    latency_ms integer null,
+    created_at timestamptz not null default timezone('utc', now())
+);
+
+create index if not exists idx_query_traces_user_created
+    on public.query_traces (user_id, created_at desc);
+
+create index if not exists idx_query_traces_session_created
+    on public.query_traces (session_id, created_at desc);
+
+alter table public.query_traces enable row level security;
+
+drop policy if exists query_traces_select_own on public.query_traces;
+create policy query_traces_select_own
+    on public.query_traces
+    for select
+    to authenticated
+    using (auth.uid() = user_id);
+
+drop policy if exists query_traces_insert_own on public.query_traces;
+create policy query_traces_insert_own
+    on public.query_traces
+    for insert
+    to authenticated
+    with check (auth.uid() = user_id);
+
+
+create table if not exists public.answer_feedback (
+    id bigint generated by default as identity primary key,
+    trace_id uuid not null references public.query_traces(trace_id) on delete cascade,
+    user_id uuid null,
+    helpful boolean null,
+    accepted boolean null,
+    reason_code text null,
+    correction_text text null,
+    promote_to_eval boolean not null default false,
+    created_at timestamptz not null default timezone('utc', now())
+);
+
+create index if not exists idx_answer_feedback_trace_created
+    on public.answer_feedback (trace_id, created_at desc);
+
+create index if not exists idx_answer_feedback_user_created
+    on public.answer_feedback (user_id, created_at desc);
+
+alter table public.answer_feedback enable row level security;
+
+drop policy if exists answer_feedback_select_own on public.answer_feedback;
+create policy answer_feedback_select_own
+    on public.answer_feedback
+    for select
+    to authenticated
+    using (auth.uid() = user_id);
+
+drop policy if exists answer_feedback_insert_own on public.answer_feedback;
+create policy answer_feedback_insert_own
+    on public.answer_feedback
+    for insert
+    to authenticated
+    with check (auth.uid() = user_id);
+
+
+create table if not exists public.graph_nodes (
+    id bigint generated by default as identity primary key,
+    user_id uuid null,
+    node_key text not null,
+    node_type text not null,
+    label text not null,
+    payload jsonb not null default '{}'::jsonb,
+    created_at timestamptz not null default timezone('utc', now()),
+    unique (user_id, node_key)
+);
+
+create index if not exists idx_graph_nodes_user_type
+    on public.graph_nodes (user_id, node_type);
+
+create index if not exists idx_graph_nodes_user_label
+    on public.graph_nodes (user_id, label);
+
+alter table public.graph_nodes enable row level security;
+
+drop policy if exists graph_nodes_select_own on public.graph_nodes;
+create policy graph_nodes_select_own
+    on public.graph_nodes
+    for select
+    to authenticated
+    using (auth.uid() = user_id);
+
+
+create table if not exists public.graph_edges (
+    id bigint generated by default as identity primary key,
+    user_id uuid null,
+    source_node_key text not null,
+    target_node_key text not null,
+    edge_type text not null,
+    weight double precision not null default 1.0,
+    payload jsonb not null default '{}'::jsonb,
+    created_at timestamptz not null default timezone('utc', now()),
+    unique (user_id, source_node_key, target_node_key, edge_type)
+);
+
+create index if not exists idx_graph_edges_user_source
+    on public.graph_edges (user_id, source_node_key);
+
+create index if not exists idx_graph_edges_user_target
+    on public.graph_edges (user_id, target_node_key);
+
+alter table public.graph_edges enable row level security;
+
+drop policy if exists graph_edges_select_own on public.graph_edges;
+create policy graph_edges_select_own
+    on public.graph_edges
+    for select
+    to authenticated
+    using (auth.uid() = user_id);

supabase/migrations/0011_admin_review_eval_workflow.sql

@@ -0,0 +1,38 @@
+alter table public.query_traces
+    add column if not exists review_state text not null default 'pending',
+    add column if not exists review_notes text null,
+    add column if not exists reviewed_at timestamptz null,
+    add column if not exists reviewed_by text null,
+    add column if not exists promoted_to_eval boolean not null default false,
+    add column if not exists document_types jsonb not null default '[]'::jsonb;
+
+create index if not exists idx_query_traces_review_state_created
+    on public.query_traces (review_state, created_at desc);
+
+alter table public.answer_feedback
+    add column if not exists review_state text not null default 'pending',
+    add column if not exists review_notes text null,
+    add column if not exists reviewed_at timestamptz null,
+    add column if not exists reviewed_by text null,
+    add column if not exists promoted_at timestamptz null;
+
+create index if not exists idx_answer_feedback_review_state_created
+    on public.answer_feedback (review_state, created_at desc);
+
+create table if not exists public.evaluation_datasets (
+    id bigint generated by default as identity primary key,
+    trace_id uuid unique null references public.query_traces(trace_id) on delete set null,
+    source text not null default 'feedback_trace',
+    question text not null,
+    gold_context_refs jsonb not null default '[]'::jsonb,
+    gold_evidence_text text null,
+    is_answerable boolean not null default true,
+    failure_modes jsonb not null default '[]'::jsonb,
+    doc_diagnostics jsonb not null default '[]'::jsonb,
+    reason_code text null,
+    is_active boolean not null default false,
+    created_at timestamptz not null default timezone('utc', now())
+);
+
+create index if not exists idx_evaluation_datasets_active_created
+    on public.evaluation_datasets (is_active, created_at desc);

supabase/migrations/0012_lock_down_evaluation_datasets.sql

@@ -0,0 +1,14 @@
+alter table public.evaluation_datasets
+    enable row level security;
+
+revoke all on public.evaluation_datasets from anon, authenticated;
+
+drop policy if exists evaluation_datasets_select_own on public.evaluation_datasets;
+drop policy if exists evaluation_datasets_insert_own on public.evaluation_datasets;
+drop policy if exists evaluation_datasets_update_own on public.evaluation_datasets;
+drop policy if exists evaluation_datasets_delete_own on public.evaluation_datasets;
+
+-- evaluation_datasets is an internal curation/evaluation table.
+-- The app reads/writes it via service-role admin/eval paths only.
+-- With RLS enabled and no anon/authenticated policies, normal clients cannot
+-- access it through PostgREST even though it lives in the public schema.

supabase/migrations/0013_backend_owned_retrieval_hardening.sql

@@ -0,0 +1,260 @@
+-- Migration 0013: backend-owned retrieval hardening
+--
+-- Goals:
+-- 1. Add a bulk chunk insert RPC for ingestion throughput.
+-- 2. Move retrieval/memory RPCs to explicit user_id scoping so the backend can
+--    call them with the service role instead of relying on browser RLS.
+-- 3. Lock internal telemetry/eval tables down to backend-only access.
+
+CREATE OR REPLACE FUNCTION public.insert_document_chunks_batch(
+    p_rows jsonb
+) RETURNS void
+    LANGUAGE plpgsql
+    SECURITY DEFINER
+    SET search_path = ''
+    AS $$
+BEGIN
+    IF p_rows IS NULL OR jsonb_typeof(p_rows) <> 'array' THEN
+        RETURN;
+    END IF;
+
+    INSERT INTO public.documents (
+        id,
+        content,
+        metadata,
+        embedding,
+        user_id,
+        node_type,
+        parent_node_id,
+        node_level
+    )
+    SELECT
+        (row->>'id')::uuid,
+        row->>'content',
+        COALESCE(row->'metadata', '{}'::jsonb),
+        (row->'embedding')::text::extensions.vector,
+        (row->>'user_id')::uuid,
+        COALESCE(NULLIF(row->>'node_type', ''), 'leaf'),
+        NULLIF(row->>'parent_node_id', '')::uuid,
+        COALESCE(NULLIF(row->>'node_level', '')::integer, 0)
+    FROM jsonb_array_elements(p_rows) AS row
+    ON CONFLICT (id) DO UPDATE
+        SET content = EXCLUDED.content,
+            metadata = EXCLUDED.metadata,
+            embedding = EXCLUDED.embedding,
+            user_id = EXCLUDED.user_id,
+            node_type = EXCLUDED.node_type,
+            parent_node_id = EXCLUDED.parent_node_id,
+            node_level = EXCLUDED.node_level;
+END;
+$$;
+
+
+CREATE OR REPLACE FUNCTION public.hybrid_search(
+    query_text text,
+    query_embedding extensions.vector,
+    match_count integer DEFAULT 10,
+    filter jsonb DEFAULT '{}'::jsonb,
+    semantic_weight double precision DEFAULT 0.7,
+    keyword_weight double precision DEFAULT 0.3,
+    p_user_id uuid DEFAULT NULL::uuid
+) RETURNS TABLE(id uuid, content text, metadata jsonb, combined_score double precision)
+    LANGUAGE plpgsql
+    SET search_path = ''
+    AS $$
+BEGIN
+    RETURN QUERY
+    WITH
+    semantic AS (
+        SELECT
+            d.id,
+            d.content,
+            d.metadata,
+            (
+                1 - (
+                    d.embedding::extensions.halfvec(2048)
+                    OPERATOR(extensions.<=>)
+                    query_embedding::extensions.halfvec(2048)
+                )
+            )::float AS score
+        FROM public.documents AS d
+        WHERE (p_user_id IS NULL OR d.user_id = p_user_id)
+          AND (filter = '{}'::jsonb OR d.metadata @> filter::jsonb)
+        ORDER BY d.embedding::extensions.halfvec(2048)
+            OPERATOR(extensions.<=>)
+            query_embedding::extensions.halfvec(2048)
+        LIMIT match_count * 3
+    ),
+    keyword AS (
+        SELECT
+            d.id,
+            d.content,
+            d.metadata,
+            pg_catalog.ts_rank(
+                pg_catalog.to_tsvector('english', d.content),
+                pg_catalog.plainto_tsquery('english', query_text)
+            )::float AS raw_score
+        FROM public.documents AS d
+        WHERE (p_user_id IS NULL OR d.user_id = p_user_id)
+          AND (filter = '{}'::jsonb OR d.metadata @> filter::jsonb)
+          AND pg_catalog.to_tsvector('english', d.content)
+              @@ pg_catalog.plainto_tsquery('english', query_text)
+        ORDER BY raw_score DESC
+        LIMIT match_count * 3
+    ),
+    keyword_norm AS (
+        SELECT
+            k.id,
+            k.content,
+            k.metadata,
+            CASE
+                WHEN max(k.raw_score) OVER () = 0 THEN 0::float
+                ELSE (k.raw_score / max(k.raw_score) OVER ())::float
+            END AS score
+        FROM keyword AS k
+    ),
+    blended AS (
+        SELECT
+            COALESCE(s.id, kn.id) AS id,
+            COALESCE(s.content, kn.content) AS content,
+            COALESCE(s.metadata, kn.metadata) AS metadata,
+            (
+                COALESCE(s.score, 0::float) * semantic_weight +
+                COALESCE(kn.score, 0::float) * keyword_weight
+            ) AS combined_score
+        FROM semantic AS s
+        FULL OUTER JOIN keyword_norm AS kn ON s.id = kn.id
+    )
+    SELECT
+        b.id,
+        b.content,
+        b.metadata,
+        b.combined_score
+    FROM blended AS b
+    ORDER BY b.combined_score DESC
+    LIMIT match_count;
+END;
+$$;
+
+
+CREATE OR REPLACE FUNCTION public.match_documents(
+    query_embedding extensions.vector,
+    match_count integer DEFAULT 5,
+    filter jsonb DEFAULT '{}'::jsonb,
+    p_user_id uuid DEFAULT NULL::uuid
+) RETURNS TABLE(id uuid, content text, metadata jsonb, similarity double precision)
+    LANGUAGE plpgsql
+    SET search_path = ''
+    AS $$
+BEGIN
+    RETURN QUERY
+    SELECT
+        d.id,
+        d.content,
+        d.metadata,
+        (
+            1 - (
+                d.embedding::extensions.halfvec(2048)
+                OPERATOR(extensions.<=>)
+                query_embedding::extensions.halfvec(2048)
+            )
+        )::float AS similarity
+    FROM public.documents AS d
+    WHERE (p_user_id IS NULL OR d.user_id = p_user_id)
+      AND (filter = '{}'::jsonb OR d.metadata @> filter::jsonb)
+    ORDER BY d.embedding::extensions.halfvec(2048)
+        OPERATOR(extensions.<=>)
+        query_embedding::extensions.halfvec(2048)
+    LIMIT match_count;
+END;
+$$;
+
+
+CREATE OR REPLACE FUNCTION public.match_memory(
+    query_embedding extensions.vector,
+    match_session_id text,
+    match_count integer DEFAULT 4,
+    p_user_id uuid DEFAULT NULL::uuid
+) RETURNS TABLE(id uuid, role text, content text, similarity double precision)
+    LANGUAGE plpgsql
+    SET search_path = ''
+    AS $$
+BEGIN
+    RETURN QUERY
+    SELECT
+        cm.id,
+        cm.role,
+        cm.content,
+        1 - (cm.embedding OPERATOR(extensions.<=>) query_embedding) AS similarity
+    FROM public.chat_memory AS cm
+    WHERE cm.session_id = match_session_id
+      AND (p_user_id IS NULL OR cm.user_id = p_user_id)
+    ORDER BY cm.embedding OPERATOR(extensions.<=>) query_embedding
+    LIMIT match_count;
+END;
+$$;
+
+
+DO $$
+BEGIN
+    IF to_regclass('public.query_traces') IS NOT NULL THEN
+        EXECUTE 'ALTER TABLE public.query_traces ENABLE ROW LEVEL SECURITY';
+        EXECUTE 'REVOKE ALL ON TABLE public.query_traces FROM anon, authenticated';
+        EXECUTE 'DROP POLICY IF EXISTS query_traces_select_own ON public.query_traces';
+        EXECUTE 'DROP POLICY IF EXISTS query_traces_insert_own ON public.query_traces';
+    END IF;
+
+    IF to_regclass('public.answer_feedback') IS NOT NULL THEN
+        EXECUTE 'ALTER TABLE public.answer_feedback ENABLE ROW LEVEL SECURITY';
+        EXECUTE 'REVOKE ALL ON TABLE public.answer_feedback FROM anon, authenticated';
+        EXECUTE 'DROP POLICY IF EXISTS answer_feedback_select_own ON public.answer_feedback';
+        EXECUTE 'DROP POLICY IF EXISTS answer_feedback_insert_own ON public.answer_feedback';
+    END IF;
+
+    IF to_regclass('public.evaluation_logs') IS NOT NULL THEN
+        EXECUTE 'ALTER TABLE public.evaluation_logs ENABLE ROW LEVEL SECURITY';
+        EXECUTE 'REVOKE ALL ON TABLE public.evaluation_logs FROM anon, authenticated';
+        EXECUTE 'DROP POLICY IF EXISTS evaluation_logs_insert_own ON public.evaluation_logs';
+        EXECUTE 'DROP POLICY IF EXISTS evaluation_logs_select_own ON public.evaluation_logs';
+    END IF;
+
+    IF to_regclass('public.intent_feedback') IS NOT NULL THEN
+        EXECUTE 'ALTER TABLE public.intent_feedback ENABLE ROW LEVEL SECURITY';
+        EXECUTE 'REVOKE ALL ON TABLE public.intent_feedback FROM anon, authenticated';
+        EXECUTE 'DROP POLICY IF EXISTS intent_feedback_select_own ON public.intent_feedback';
+        EXECUTE 'DROP POLICY IF EXISTS intent_feedback_insert_own ON public.intent_feedback';
+    END IF;
+
+    IF to_regclass('public.rerank_feedback') IS NOT NULL THEN
+        EXECUTE 'ALTER TABLE public.rerank_feedback ENABLE ROW LEVEL SECURITY';
+        EXECUTE 'REVOKE ALL ON TABLE public.rerank_feedback FROM anon, authenticated';
+        EXECUTE 'DROP POLICY IF EXISTS rerank_feedback_select_own ON public.rerank_feedback';
+    END IF;
+
+    IF to_regclass('public.graph_nodes') IS NOT NULL THEN
+        EXECUTE 'ALTER TABLE public.graph_nodes ENABLE ROW LEVEL SECURITY';
+        EXECUTE 'REVOKE ALL ON TABLE public.graph_nodes FROM anon, authenticated';
+        EXECUTE 'DROP POLICY IF EXISTS graph_nodes_select_own ON public.graph_nodes';
+    END IF;
+
+    IF to_regclass('public.graph_edges') IS NOT NULL THEN
+        EXECUTE 'ALTER TABLE public.graph_edges ENABLE ROW LEVEL SECURITY';
+        EXECUTE 'REVOKE ALL ON TABLE public.graph_edges FROM anon, authenticated';
+        EXECUTE 'DROP POLICY IF EXISTS graph_edges_select_own ON public.graph_edges';
+    END IF;
+
+    IF to_regclass('public.category_centroids') IS NOT NULL THEN
+        EXECUTE 'ALTER TABLE public.category_centroids ENABLE ROW LEVEL SECURITY';
+        EXECUTE 'REVOKE ALL ON TABLE public.category_centroids FROM anon, authenticated';
+    END IF;
+
+    IF to_regclass('public.ingestion_retry_logs') IS NOT NULL THEN
+        EXECUTE 'ALTER TABLE public.ingestion_retry_logs ENABLE ROW LEVEL SECURITY';
+        EXECUTE 'REVOKE ALL ON TABLE public.ingestion_retry_logs FROM anon, authenticated';
+        EXECUTE 'DROP POLICY IF EXISTS ingestion_retry_logs_select_own ON public.ingestion_retry_logs';
+        EXECUTE 'DROP POLICY IF EXISTS ingestion_retry_logs_insert_own ON public.ingestion_retry_logs';
+        EXECUTE 'DROP POLICY IF EXISTS ingestion_retry_logs_update_own ON public.ingestion_retry_logs';
+        EXECUTE 'DROP POLICY IF EXISTS ingestion_retry_logs_delete_own ON public.ingestion_retry_logs';
+    END IF;
+END;
+$$;

supabase/migrations/0014_drop_legacy_category_centroid_policies.sql

@@ -0,0 +1,20 @@
+-- Migration 0014: drop legacy category_centroids user-facing RLS policies
+--
+-- 0013 moved centroid access to backend-owned service-role calls with
+-- explicit user_id filtering, but it did not remove the older auth.uid()
+-- policies. Those stale policies keep Security Advisor warning about
+-- anonymous access on public.category_centroids and also keep schema dumps
+-- out of sync with the intended access model.
+
+DO $$
+BEGIN
+    IF to_regclass('public.category_centroids') IS NOT NULL THEN
+        EXECUTE 'ALTER TABLE public.category_centroids ENABLE ROW LEVEL SECURITY';
+        EXECUTE 'REVOKE ALL ON TABLE public.category_centroids FROM anon, authenticated';
+        EXECUTE 'DROP POLICY IF EXISTS centroids_select_own ON public.category_centroids';
+        EXECUTE 'DROP POLICY IF EXISTS centroids_insert_own ON public.category_centroids';
+        EXECUTE 'DROP POLICY IF EXISTS centroids_update_own ON public.category_centroids';
+        EXECUTE 'DROP POLICY IF EXISTS centroids_delete_own ON public.category_centroids';
+    END IF;
+END
+$$;

supabase/migrations/0015_ingested_file_identity_json.sql

@@ -0,0 +1,2 @@
+alter table public.ingested_files
+  add column if not exists identity_json jsonb not null default '{}'::jsonb;

supabase/migrations/0016_ingestion_file_hash_checkpoints.sql

@@ -0,0 +1,5 @@
+ALTER TABLE public.ingestion_retry_logs
+ADD COLUMN IF NOT EXISTS file_hash text;
+
+CREATE INDEX IF NOT EXISTS ingestion_retry_logs_user_file_event_idx
+ON public.ingestion_retry_logs (user_id, file_hash, event_type, created_at DESC);

supabase/schema_backup.sql

@@ -1,74 +1,60 @@
 --
 -- PostgreSQL database dump
 --
-
--- Dumped from database version 17.6
--- Dumped by pg_dump version 18.3
-
-SET statement_timeout = 0;
-SET lock_timeout = 0;
-SET idle_in_transaction_session_timeout = 0;
-SET transaction_timeout = 0;
-SET client_encoding = 'UTF8';
-SET standard_conforming_strings = on;
-SELECT pg_catalog.set_config('search_path', '', false);
-SET check_function_bodies = false;
-SET xmloption = content;
-SET client_min_messages = warning;
-SET row_security = off;
-
---
--- Name: public; Type: SCHEMA; Schema: -; Owner: -
---
-
-CREATE SCHEMA IF NOT EXISTS public;
-
-
---
--- Name: SCHEMA public; Type: COMMENT; Schema: -; Owner: -
---
-
-COMMENT ON SCHEMA public IS 'standard public schema';
-
-
---
--- Name: _trg_refresh_mv_document_types(); Type: FUNCTION; Schema: public; Owner: -
---
-
--- CREATE FUNCTION public._trg_refresh_mv_document_types() RETURNS trigger
---     LANGUAGE plpgsql
---     AS $$
--- begin
---     -- Fire-and-forget: refresh in background via pg_notify
---     -- (avoids blocking the INSERT transaction itself)
---     perform pg_notify('refresh_mv', 'document_types');
---     return new;
--- end;
--- $$;
-
-
---
--- Name: _trg_set_updated_at(); Type: FUNCTION; Schema: public; Owner: -
---
-
+
+\restrict 32urOXpOnsQS0zoo7jGTkIs0BeRgGPyJVLWPDJ6IexS9GSsM4lpkxJaAg6FM0Ua
+
+-- Dumped from database version 17.6
+-- Dumped by pg_dump version 18.3
+
+SET statement_timeout = 0;
+SET lock_timeout = 0;
+SET idle_in_transaction_session_timeout = 0;
+SET transaction_timeout = 0;
+SET client_encoding = 'UTF8';
+SET standard_conforming_strings = on;
+SELECT pg_catalog.set_config('search_path', '', false);
+SET check_function_bodies = false;
+SET xmloption = content;
+SET client_min_messages = warning;
+SET row_security = off;
+
+--
+-- Name: public; Type: SCHEMA; Schema: -; Owner: -
+--
+
+CREATE SCHEMA public;
+
+
+--
+-- Name: SCHEMA public; Type: COMMENT; Schema: -; Owner: -
+--
+
+COMMENT ON SCHEMA public IS 'standard public schema';
+
+
+--
+-- Name: _trg_set_updated_at(); Type: FUNCTION; Schema: public; Owner: -
+--
+
 CREATE FUNCTION public._trg_set_updated_at() RETURNS trigger
     LANGUAGE plpgsql
-    SET search_path = ''
-    AS $$
-begin
+    SET search_path TO ''
+    AS $$
+begin
     new.updated_at = pg_catalog.now();
-    return new;
-end;
-$$;
-
-
---
--- Name: get_document_types(); Type: FUNCTION; Schema: public; Owner: -
---
-
+    return new;
+end;
+$$;
+
+
+--
+-- Name: get_document_types(); Type: FUNCTION; Schema: public; Owner: -
+--
+
 CREATE FUNCTION public.get_document_types() RETURNS TABLE(document_type text)
     LANGUAGE sql STABLE
-    SET search_path = ''
+    SET search_path TO ''
     AS $$
     select distinct f.document_type
     from public.ingested_files as f
@@ -76,23 +62,25 @@ CREATE FUNCTION public.get_document_types() RETURNS TABLE(document_type text)
       and f.document_type is not null
       and f.document_type <> 'unknown'
     order by f.document_type;
-$$;
-
-
---
+$$;
+
+
+--
 -- Name: hybrid_search(text, extensions.vector, integer, jsonb, double precision, double precision); Type: FUNCTION; Schema: public; Owner: -
---
-
+--
+
 CREATE FUNCTION public.hybrid_search(query_text text, query_embedding extensions.vector, match_count integer DEFAULT 10, filter jsonb DEFAULT '{}'::jsonb, semantic_weight double precision DEFAULT 0.7, keyword_weight double precision DEFAULT 0.3) RETURNS TABLE(id uuid, content text, metadata jsonb, combined_score double precision)
     LANGUAGE plpgsql
-    SET search_path = ''
+    SET search_path TO ''
     AS $$
-begin
-    return query
-    with
-    semantic as (
-        select
-            d.id, d.content, d.metadata,
+begin
+    return query
+    with
+    semantic as (
+        select
+            d.id,
+            d.content,
+            d.metadata,
             (
                 1 - (
                     d.embedding::extensions.halfvec(2048)
@@ -101,937 +89,1390 @@ begin
                 )
             )::float as score
         from public.documents d
-        where (filter = '{}'::jsonb or d.metadata @> filter::jsonb)
+        where (filter = '{}'::jsonb or d.metadata @> filter::jsonb)
         order by d.embedding::extensions.halfvec(2048)
             OPERATOR(extensions.<=>)
             query_embedding::extensions.halfvec(2048)
-        limit match_count * 3
-    ),
-    keyword as (
-        select
-            d.id, d.content, d.metadata,
+        limit match_count * 3
+    ),
+    keyword as (
+        select
+            d.id,
+            d.content,
+            d.metadata,
             pg_catalog.ts_rank(
                 pg_catalog.to_tsvector('english', d.content),
                 pg_catalog.plainto_tsquery('english', query_text)
-            )::float as raw_score
+            )::float as raw_score
         from public.documents d
-        where (filter = '{}'::jsonb or d.metadata @> filter::jsonb)
+        where (filter = '{}'::jsonb or d.metadata @> filter::jsonb)
           and pg_catalog.to_tsvector('english', d.content) @@ pg_catalog.plainto_tsquery('english', query_text)
-        order by raw_score desc
-        limit match_count * 3
-    ),
-    keyword_norm as (
-        select k.id, k.content, k.metadata,
-            case
-                when max(k.raw_score) over () = 0 then 0::float
-                else (k.raw_score / max(k.raw_score) over ())::float
-            end as score
-        from keyword k
-    ),
-    blended as (
-        select
-            coalesce(s.id,       kn.id)       as id,
-            coalesce(s.content,  kn.content)  as content,
-            coalesce(s.metadata, kn.metadata) as metadata,
-            (
-                coalesce(s.score,  0::float) * semantic_weight +
-                coalesce(kn.score, 0::float) * keyword_weight
-            ) as combined_score
-        from semantic s
-        full outer join keyword_norm kn on s.id = kn.id
-    )
-    select b.id, b.content, b.metadata, b.combined_score
-    from blended b
-    order by b.combined_score desc
-    limit match_count;
-end;
-$$;
-
-
---
+        order by raw_score desc
+        limit match_count * 3
+    ),
+    keyword_norm as (
+        select
+            k.id,
+            k.content,
+            k.metadata,
+            case
+                when max(k.raw_score) over () = 0 then 0::float
+                else (k.raw_score / max(k.raw_score) over ())::float
+            end as score
+        from keyword k
+    ),
+    blended as (
+        select
+            coalesce(s.id, kn.id) as id,
+            coalesce(s.content, kn.content) as content,
+            coalesce(s.metadata, kn.metadata) as metadata,
+            (
+                coalesce(s.score, 0::float) * semantic_weight +
+                coalesce(kn.score, 0::float) * keyword_weight
+            ) as combined_score
+        from semantic s
+        full outer join keyword_norm kn on s.id = kn.id
+    )
+    select
+        b.id,
+        b.content,
+        b.metadata,
+        b.combined_score
+    from blended b
+    order by b.combined_score desc
+    limit match_count;
+end;
+$$;
+
+
+--
+-- Name: hybrid_search(text, extensions.vector, integer, jsonb, double precision, double precision, uuid); Type: FUNCTION; Schema: public; Owner: -
+--
+
+CREATE FUNCTION public.hybrid_search(query_text text, query_embedding extensions.vector, match_count integer DEFAULT 10, filter jsonb DEFAULT '{}'::jsonb, semantic_weight double precision DEFAULT 0.7, keyword_weight double precision DEFAULT 0.3, p_user_id uuid DEFAULT NULL::uuid) RETURNS TABLE(id uuid, content text, metadata jsonb, combined_score double precision)
+    LANGUAGE plpgsql
+    SET search_path TO ''
+    AS $$
+BEGIN
+    RETURN QUERY
+    WITH
+    semantic AS (
+        SELECT
+            d.id,
+            d.content,
+            d.metadata,
+            (
+                1 - (
+                    d.embedding::extensions.halfvec(2048)
+                    OPERATOR(extensions.<=>)
+                    query_embedding::extensions.halfvec(2048)
+                )
+            )::float AS score
+        FROM public.documents AS d
+        WHERE (p_user_id IS NULL OR d.user_id = p_user_id)
+          AND (filter = '{}'::jsonb OR d.metadata @> filter::jsonb)
+        ORDER BY d.embedding::extensions.halfvec(2048)
+            OPERATOR(extensions.<=>)
+            query_embedding::extensions.halfvec(2048)
+        LIMIT match_count * 3
+    ),
+    keyword AS (
+        SELECT
+            d.id,
+            d.content,
+            d.metadata,
+            pg_catalog.ts_rank(
+                pg_catalog.to_tsvector('english', d.content),
+                pg_catalog.plainto_tsquery('english', query_text)
+            )::float AS raw_score
+        FROM public.documents AS d
+        WHERE (p_user_id IS NULL OR d.user_id = p_user_id)
+          AND (filter = '{}'::jsonb OR d.metadata @> filter::jsonb)
+          AND pg_catalog.to_tsvector('english', d.content)
+              @@ pg_catalog.plainto_tsquery('english', query_text)
+        ORDER BY raw_score DESC
+        LIMIT match_count * 3
+    ),
+    keyword_norm AS (
+        SELECT
+            k.id,
+            k.content,
+            k.metadata,
+            CASE
+                WHEN max(k.raw_score) OVER () = 0 THEN 0::float
+                ELSE (k.raw_score / max(k.raw_score) OVER ())::float
+            END AS score
+        FROM keyword AS k
+    ),
+    blended AS (
+        SELECT
+            COALESCE(s.id, kn.id) AS id,
+            COALESCE(s.content, kn.content) AS content,
+            COALESCE(s.metadata, kn.metadata) AS metadata,
+            (
+                COALESCE(s.score, 0::float) * semantic_weight +
+                COALESCE(kn.score, 0::float) * keyword_weight
+            ) AS combined_score
+        FROM semantic AS s
+        FULL OUTER JOIN keyword_norm AS kn ON s.id = kn.id
+    )
+    SELECT
+        b.id,
+        b.content,
+        b.metadata,
+        b.combined_score
+    FROM blended AS b
+    ORDER BY b.combined_score DESC
+    LIMIT match_count;
+END;
+$$;
+
+
+--
 -- Name: insert_document_chunk(uuid, text, jsonb, extensions.vector, uuid); Type: FUNCTION; Schema: public; Owner: -
---
-
+--
+
 CREATE FUNCTION public.insert_document_chunk(p_id uuid, p_content text, p_metadata jsonb, p_embedding extensions.vector, p_user_id uuid) RETURNS void
     LANGUAGE plpgsql SECURITY DEFINER
-    SET search_path = ''
+    SET search_path TO ''
     AS $$
-BEGIN
+BEGIN
   INSERT INTO public.documents (id, content, metadata, embedding, user_id)
-  VALUES (p_id, p_content, p_metadata, p_embedding, p_user_id)
-  ON CONFLICT (id) DO UPDATE
-    SET content   = EXCLUDED.content,
-        metadata  = EXCLUDED.metadata,
-        embedding = EXCLUDED.embedding;
-END;
-$$;
-
-
---
+  VALUES (p_id, p_content, p_metadata, p_embedding, p_user_id)
+  ON CONFLICT (id) DO UPDATE
+    SET content   = EXCLUDED.content,
+        metadata  = EXCLUDED.metadata,
+        embedding = EXCLUDED.embedding;
+END;
+$$;
+
+
+--
 -- Name: insert_document_chunk(uuid, text, jsonb, extensions.vector, uuid, text, uuid, integer); Type: FUNCTION; Schema: public; Owner: -
---
-
+--
+
 CREATE FUNCTION public.insert_document_chunk(p_id uuid, p_content text, p_metadata jsonb, p_embedding extensions.vector, p_user_id uuid, p_node_type text DEFAULT 'leaf'::text, p_parent_node_id uuid DEFAULT NULL::uuid, p_node_level integer DEFAULT 0) RETURNS void
     LANGUAGE plpgsql SECURITY DEFINER
-    SET search_path = ''
+    SET search_path TO ''
     AS $$
-BEGIN
+BEGIN
   INSERT INTO public.documents (
-      id, content, metadata, embedding, user_id,
-      node_type, parent_node_id, node_level
-  )
-  VALUES (
-      p_id, p_content, p_metadata, p_embedding, p_user_id,
-      p_node_type, p_parent_node_id, p_node_level
-  )
-  ON CONFLICT (id) DO UPDATE
-    SET content        = EXCLUDED.content,
-        metadata       = EXCLUDED.metadata,
-        embedding      = EXCLUDED.embedding,
-        node_type      = EXCLUDED.node_type,
-        parent_node_id = EXCLUDED.parent_node_id,
-        node_level     = EXCLUDED.node_level;
-END;
-$$;
-
-
---
+      id, content, metadata, embedding, user_id,
+      node_type, parent_node_id, node_level
+  )
+  VALUES (
+      p_id, p_content, p_metadata, p_embedding, p_user_id,
+      p_node_type, p_parent_node_id, p_node_level
+  )
+  ON CONFLICT (id) DO UPDATE
+    SET content        = EXCLUDED.content,
+        metadata       = EXCLUDED.metadata,
+        embedding      = EXCLUDED.embedding,
+        node_type      = EXCLUDED.node_type,
+        parent_node_id = EXCLUDED.parent_node_id,
+        node_level     = EXCLUDED.node_level;
+END;
+$$;
+
+
+--
+-- Name: insert_document_chunks_batch(jsonb); Type: FUNCTION; Schema: public; Owner: -
+--
+
+CREATE FUNCTION public.insert_document_chunks_batch(p_rows jsonb) RETURNS void
+    LANGUAGE plpgsql SECURITY DEFINER
+    SET search_path TO ''
+    AS $$
+BEGIN
+    IF p_rows IS NULL OR jsonb_typeof(p_rows) <> 'array' THEN
+        RETURN;
+    END IF;
+
+    INSERT INTO public.documents (
+        id,
+        content,
+        metadata,
+        embedding,
+        user_id,
+        node_type,
+        parent_node_id,
+        node_level
+    )
+    SELECT
+        (row->>'id')::uuid,
+        row->>'content',
+        COALESCE(row->'metadata', '{}'::jsonb),
+        (row->'embedding')::text::extensions.vector,
+        (row->>'user_id')::uuid,
+        COALESCE(NULLIF(row->>'node_type', ''), 'leaf'),
+        NULLIF(row->>'parent_node_id', '')::uuid,
+        COALESCE(NULLIF(row->>'node_level', '')::integer, 0)
+    FROM jsonb_array_elements(p_rows) AS row
+    ON CONFLICT (id) DO UPDATE
+        SET content = EXCLUDED.content,
+            metadata = EXCLUDED.metadata,
+            embedding = EXCLUDED.embedding,
+            user_id = EXCLUDED.user_id,
+            node_type = EXCLUDED.node_type,
+            parent_node_id = EXCLUDED.parent_node_id,
+            node_level = EXCLUDED.node_level;
+END;
+$$;
+
+
+--
 -- Name: match_documents(extensions.vector, integer, jsonb); Type: FUNCTION; Schema: public; Owner: -
---
-
+--
+
 CREATE FUNCTION public.match_documents(query_embedding extensions.vector, match_count integer DEFAULT 5, filter jsonb DEFAULT '{}'::jsonb) RETURNS TABLE(id uuid, content text, metadata jsonb, similarity double precision)
     LANGUAGE plpgsql
-    SET search_path = ''
+    SET search_path TO ''
     AS $$
-begin
-    return query
-    select
-        d.id,
-        d.content,
-        d.metadata,
-          (
-              1 - (
-                  d.embedding::extensions.halfvec(2048)
-                  OPERATOR(extensions.<=>)
-                  query_embedding::extensions.halfvec(2048)
-              )
-          )::float as similarity
+begin
+    return query
+    select
+        d.id,
+        d.content,
+        d.metadata,
+        (
+            1 - (
+                d.embedding::extensions.halfvec(2048)
+                OPERATOR(extensions.<=>)
+                query_embedding::extensions.halfvec(2048)
+            )
+        )::float as similarity
     from public.documents d
-    where (filter = '{}'::jsonb or d.metadata @> filter::jsonb)
-      order by d.embedding::extensions.halfvec(2048)
-          OPERATOR(extensions.<=>)
-          query_embedding::extensions.halfvec(2048)
-    limit match_count;
-end;
-$$;
-
-
---
+    where (filter = '{}'::jsonb or d.metadata @> filter::jsonb)
+    order by d.embedding::extensions.halfvec(2048)
+        OPERATOR(extensions.<=>)
+        query_embedding::extensions.halfvec(2048)
+    limit match_count;
+end;
+$$;
+
+
+--
+-- Name: match_documents(extensions.vector, integer, jsonb, uuid); Type: FUNCTION; Schema: public; Owner: -
+--
+
+CREATE FUNCTION public.match_documents(query_embedding extensions.vector, match_count integer DEFAULT 5, filter jsonb DEFAULT '{}'::jsonb, p_user_id uuid DEFAULT NULL::uuid) RETURNS TABLE(id uuid, content text, metadata jsonb, similarity double precision)
+    LANGUAGE plpgsql
+    SET search_path TO ''
+    AS $$
+BEGIN
+    RETURN QUERY
+    SELECT
+        d.id,
+        d.content,
+        d.metadata,
+        (
+            1 - (
+                d.embedding::extensions.halfvec(2048)
+                OPERATOR(extensions.<=>)
+                query_embedding::extensions.halfvec(2048)
+            )
+        )::float AS similarity
+    FROM public.documents AS d
+    WHERE (p_user_id IS NULL OR d.user_id = p_user_id)
+      AND (filter = '{}'::jsonb OR d.metadata @> filter::jsonb)
+    ORDER BY d.embedding::extensions.halfvec(2048)
+        OPERATOR(extensions.<=>)
+        query_embedding::extensions.halfvec(2048)
+    LIMIT match_count;
+END;
+$$;
+
+
+--
 -- Name: match_memory(extensions.vector, text, integer); Type: FUNCTION; Schema: public; Owner: -
---
-
+--
+
 CREATE FUNCTION public.match_memory(query_embedding extensions.vector, match_session_id text, match_count integer DEFAULT 4) RETURNS TABLE(id uuid, role text, content text, similarity double precision)
     LANGUAGE plpgsql
-    SET search_path = ''
+    SET search_path TO ''
+    AS $$
+BEGIN
+    RETURN QUERY
+    SELECT
+        cm.id,
+        cm.role,
+        cm.content,
+        1 - (cm.embedding OPERATOR(extensions.<=>) query_embedding) AS similarity
+    FROM public.chat_memory AS cm
+    WHERE cm.session_id = match_session_id
+    ORDER BY cm.embedding OPERATOR(extensions.<=>) query_embedding
+    LIMIT match_count;
+END;
+$$;
+
+
+--
+-- Name: match_memory(extensions.vector, text, integer, uuid); Type: FUNCTION; Schema: public; Owner: -
+--
+
+CREATE FUNCTION public.match_memory(query_embedding extensions.vector, match_session_id text, match_count integer DEFAULT 4, p_user_id uuid DEFAULT NULL::uuid) RETURNS TABLE(id uuid, role text, content text, similarity double precision)
+    LANGUAGE plpgsql
+    SET search_path TO ''
     AS $$
-BEGIN
-    RETURN QUERY
-    SELECT
+BEGIN
+    RETURN QUERY
+    SELECT
         cm.id,
         cm.role,
         cm.content,
         1 - (cm.embedding OPERATOR(extensions.<=>) query_embedding) AS similarity
     FROM public.chat_memory AS cm
     WHERE cm.session_id = match_session_id
+      AND (p_user_id IS NULL OR cm.user_id = p_user_id)
     ORDER BY cm.embedding OPERATOR(extensions.<=>) query_embedding
-    LIMIT match_count;
-END;
-$$;
-
-
---
--- Name: refresh_document_types_mv(); Type: FUNCTION; Schema: public; Owner: -
---
-
--- CREATE FUNCTION public.refresh_document_types_mv() RETURNS void
---     LANGUAGE plpgsql
---     AS $$
--- begin
---     refresh materialized view concurrently mv_document_types;
--- end;
--- $$;
-
-
-SET default_tablespace = '';
-
-SET default_table_access_method = heap;
-
---
--- Name: category_centroids; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.category_centroids (
-    id uuid DEFAULT gen_random_uuid() NOT NULL,
-    document_type text NOT NULL,
-    centroid_vector double precision[] NOT NULL,
-    document_count integer DEFAULT 1,
-    created_at timestamp with time zone DEFAULT now(),
-    updated_at timestamp with time zone DEFAULT now(),
-    user_id uuid DEFAULT auth.uid()
-);
-
-
---
--- Name: chat_memory; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.chat_memory (
-    id uuid DEFAULT extensions.uuid_generate_v4() NOT NULL,
-    session_id text NOT NULL,
-    role text NOT NULL,
-    content text NOT NULL,
+    LIMIT match_count;
+END;
+$$;
+
+
+--
+-- Name: rls_auto_enable(); Type: FUNCTION; Schema: public; Owner: -
+--
+
+CREATE FUNCTION public.rls_auto_enable() RETURNS event_trigger
+    LANGUAGE plpgsql SECURITY DEFINER
+    SET search_path TO 'pg_catalog'
+    AS $$
+DECLARE
+  cmd record;
+BEGIN
+  FOR cmd IN
+    SELECT *
+    FROM pg_event_trigger_ddl_commands()
+    WHERE command_tag IN ('CREATE TABLE', 'CREATE TABLE AS', 'SELECT INTO')
+      AND object_type IN ('table','partitioned table')
+  LOOP
+     IF cmd.schema_name IS NOT NULL AND cmd.schema_name IN ('public') AND cmd.schema_name NOT IN ('pg_catalog','information_schema') AND cmd.schema_name NOT LIKE 'pg_toast%' AND cmd.schema_name NOT LIKE 'pg_temp%' THEN
+      BEGIN
+        EXECUTE format('alter table if exists %s enable row level security', cmd.object_identity);
+        RAISE LOG 'rls_auto_enable: enabled RLS on %', cmd.object_identity;
+      EXCEPTION
+        WHEN OTHERS THEN
+          RAISE LOG 'rls_auto_enable: failed to enable RLS on %', cmd.object_identity;
+      END;
+     ELSE
+        RAISE LOG 'rls_auto_enable: skip % (either system schema or not in enforced list: %.)', cmd.object_identity, cmd.schema_name;
+     END IF;
+  END LOOP;
+END;
+$$;
+
+
+SET default_tablespace = '';
+
+SET default_table_access_method = heap;
+
+--
+-- Name: answer_feedback; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.answer_feedback (
+    id bigint NOT NULL,
+    trace_id uuid NOT NULL,
+    user_id uuid,
+    helpful boolean,
+    accepted boolean,
+    reason_code text,
+    correction_text text,
+    promote_to_eval boolean DEFAULT false NOT NULL,
+    created_at timestamp with time zone DEFAULT timezone('utc'::text, now()) NOT NULL,
+    review_state text DEFAULT 'pending'::text NOT NULL,
+    review_notes text,
+    reviewed_at timestamp with time zone,
+    reviewed_by text,
+    promoted_at timestamp with time zone
+);
+
+
+--
+-- Name: answer_feedback_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+ALTER TABLE public.answer_feedback ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY (
+    SEQUENCE NAME public.answer_feedback_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1
+);
+
+
+--
+-- Name: category_centroids; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.category_centroids (
+    id uuid DEFAULT gen_random_uuid() NOT NULL,
+    document_type text NOT NULL,
+    centroid_vector double precision[] NOT NULL,
+    document_count integer DEFAULT 1,
+    created_at timestamp with time zone DEFAULT now(),
+    updated_at timestamp with time zone DEFAULT now(),
+    user_id uuid DEFAULT auth.uid()
+);
+
+
+--
+-- Name: chat_memory; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.chat_memory (
+    id uuid DEFAULT extensions.uuid_generate_v4() NOT NULL,
+    session_id text NOT NULL,
+    role text NOT NULL,
+    content text NOT NULL,
     embedding extensions.vector(2048),
-    created_at timestamp with time zone DEFAULT timezone('utc'::text, now()),
-    user_id uuid DEFAULT auth.uid()
-);
-
-
---
--- Name: document_trees; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.document_trees (
-    file_hash text NOT NULL,
-    user_id uuid NOT NULL,
-    tree_json jsonb NOT NULL,
-    created_at timestamp with time zone DEFAULT timezone('utc'::text, now())
-);
-
-
---
--- Name: documents; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.documents (
-    id uuid DEFAULT gen_random_uuid() NOT NULL,
-    content text,
-    metadata jsonb,
+    created_at timestamp with time zone DEFAULT timezone('utc'::text, now()),
+    user_id uuid DEFAULT auth.uid()
+);
+
+
+--
+-- Name: document_trees; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.document_trees (
+    file_hash text NOT NULL,
+    user_id uuid NOT NULL,
+    tree_json jsonb NOT NULL,
+    created_at timestamp with time zone DEFAULT timezone('utc'::text, now())
+);
+
+
+--
+-- Name: documents; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.documents (
+    id uuid DEFAULT gen_random_uuid() NOT NULL,
+    content text,
+    metadata jsonb,
     embedding extensions.vector(2048),
-    user_id uuid DEFAULT auth.uid(),
-    node_type text DEFAULT 'leaf'::text,
-    parent_node_id uuid,
-    node_level integer DEFAULT 0
-);
-
-
---
--- Name: evaluation_logs; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.evaluation_logs (
-    id uuid DEFAULT gen_random_uuid() NOT NULL,
-    run_label text,
-    evaluated_at timestamp with time zone,
-    alpha double precision,
-    k integer,
-    question text,
-    is_answerable boolean,
-    precision_at_k double precision,
-    faithfulness_proxy double precision,
-    relevance_proxy double precision,
-    local_reward double precision,
-    llm_judge_score double precision,
-    judge_a_verdict boolean,
-    judge_b_verdict boolean,
-    judge_a_model text,
-    judge_b_model text,
-    calibration_score double precision,
-    final_score double precision,
-    requires_manual_review boolean DEFAULT false,
-    disagreement_note text DEFAULT ''::text,
-    user_id uuid
-);
-
-
---
--- Name: ingested_files; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.ingested_files (
-    id uuid DEFAULT gen_random_uuid() NOT NULL,
-    file_hash text NOT NULL,
-    filename text NOT NULL,
-    document_type text,
-    chunk_count integer DEFAULT 0,
-    ingested_at timestamp with time zone DEFAULT now(),
-    user_id uuid DEFAULT auth.uid(),
-    user_overridden boolean DEFAULT false
-);
-
-
---
--- Name: ingestion_retry_logs; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.ingestion_retry_logs (
-    id bigint NOT NULL,
-    created_at timestamp with time zone DEFAULT now() NOT NULL,
-    user_id uuid,
-    batch_num integer NOT NULL,
-    total_batches integer NOT NULL,
-    attempt integer NOT NULL,
-    event_type text NOT NULL,
-    message text,
-    sleep_s double precision DEFAULT 0
-);
-
-
---
--- Name: ingestion_retry_logs_id_seq; Type: SEQUENCE; Schema: public; Owner: -
---
-
-CREATE SEQUENCE public.ingestion_retry_logs_id_seq
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: ingestion_retry_logs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
---
-
-ALTER SEQUENCE public.ingestion_retry_logs_id_seq OWNED BY public.ingestion_retry_logs.id;
-
-
---
--- Name: intent_feedback; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.intent_feedback (
-    id bigint NOT NULL,
-    user_id uuid,
-    query text NOT NULL,
-    has_category boolean DEFAULT false NOT NULL,
-    has_history boolean DEFAULT false NOT NULL,
-    label integer NOT NULL,
-    created_at timestamp with time zone DEFAULT now() NOT NULL,
-    CONSTRAINT intent_feedback_label_check CHECK ((label = ANY (ARRAY[0, 1])))
-);
-
-
---
--- Name: intent_feedback_id_seq; Type: SEQUENCE; Schema: public; Owner: -
---
-
-CREATE SEQUENCE public.intent_feedback_id_seq
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: intent_feedback_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
---
-
-ALTER SEQUENCE public.intent_feedback_id_seq OWNED BY public.intent_feedback.id;
-
-
---
--- Name: mv_document_types; Type: MATERIALIZED VIEW; Schema: public; Owner: -
---
-
--- CREATE MATERIALIZED VIEW public.mv_document_types AS
---  SELECT DISTINCT (metadata ->> 'document_type'::text) AS document_type
---    FROM public.documents
---   WHERE (((metadata ->> 'document_type'::text) IS NOT NULL) AND ((metadata ->> 'document_type'::text) <> 'unknown'::text))
---   ORDER BY (metadata ->> 'document_type'::text)
---   WITH NO DATA;
-
-
---
--- Name: rerank_feedback; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.rerank_feedback (
-    id bigint NOT NULL,
-    user_id uuid,
-    query_hash text NOT NULL,
-    chunk_id uuid,
-    chunk_hash text NOT NULL,
-    document_type text,
-    cohere_score real NOT NULL,
-    was_selected boolean NOT NULL,
-    created_at timestamp with time zone DEFAULT now() NOT NULL,
-    query_text text,
-    chunk_text text
-);
-
-
---
--- Name: rerank_feedback_id_seq; Type: SEQUENCE; Schema: public; Owner: -
---
-
-CREATE SEQUENCE public.rerank_feedback_id_seq
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: rerank_feedback_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
---
-
-ALTER SEQUENCE public.rerank_feedback_id_seq OWNED BY public.rerank_feedback.id;
-
-
---
--- Name: ingestion_retry_logs id; Type: DEFAULT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.ingestion_retry_logs ALTER COLUMN id SET DEFAULT nextval('public.ingestion_retry_logs_id_seq'::regclass);
-
-
---
--- Name: intent_feedback id; Type: DEFAULT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.intent_feedback ALTER COLUMN id SET DEFAULT nextval('public.intent_feedback_id_seq'::regclass);
-
-
---
--- Name: rerank_feedback id; Type: DEFAULT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.rerank_feedback ALTER COLUMN id SET DEFAULT nextval('public.rerank_feedback_id_seq'::regclass);
-
-
---
--- Name: category_centroids category_centroids_document_type_key; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.category_centroids
-    ADD CONSTRAINT category_centroids_document_type_key UNIQUE (document_type);
-
-
---
--- Name: category_centroids category_centroids_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.category_centroids
-    ADD CONSTRAINT category_centroids_pkey PRIMARY KEY (id);
-
-
---
--- Name: chat_memory chat_memory_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.chat_memory
-    ADD CONSTRAINT chat_memory_pkey PRIMARY KEY (id);
-
-
---
--- Name: document_trees document_trees_user_file_hash_key; Type: CONSTRAINT; Schema: public; Owner: -
+    user_id uuid DEFAULT auth.uid(),
+    node_type text DEFAULT 'leaf'::text,
+    parent_node_id uuid,
+    node_level integer DEFAULT 0
+);
+
+
+--
+-- Name: evaluation_datasets; Type: TABLE; Schema: public; Owner: -
 --
 
-ALTER TABLE ONLY public.document_trees
-    ADD CONSTRAINT document_trees_user_file_hash_key UNIQUE (user_id, file_hash);
-
-
---
--- Name: documents documents_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.documents
-    ADD CONSTRAINT documents_pkey PRIMARY KEY (id);
-
-
---
--- Name: evaluation_logs evaluation_logs_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.evaluation_logs
-    ADD CONSTRAINT evaluation_logs_pkey PRIMARY KEY (id);
-
-
---
--- Name: ingested_files ingested_files_user_file_hash_key; Type: CONSTRAINT; Schema: public; Owner: -
+CREATE TABLE public.evaluation_datasets (
+    id bigint NOT NULL,
+    trace_id uuid,
+    source text DEFAULT 'feedback_trace'::text NOT NULL,
+    question text NOT NULL,
+    gold_context_refs jsonb DEFAULT '[]'::jsonb NOT NULL,
+    gold_evidence_text text,
+    is_answerable boolean DEFAULT true NOT NULL,
+    failure_modes jsonb DEFAULT '[]'::jsonb NOT NULL,
+    doc_diagnostics jsonb DEFAULT '[]'::jsonb NOT NULL,
+    reason_code text,
+    is_active boolean DEFAULT false NOT NULL,
+    created_at timestamp with time zone DEFAULT timezone('utc'::text, now()) NOT NULL
+);
+
+
+--
+-- Name: evaluation_datasets_id_seq; Type: SEQUENCE; Schema: public; Owner: -
 --
 
-ALTER TABLE ONLY public.ingested_files
-    ADD CONSTRAINT ingested_files_user_file_hash_key UNIQUE (user_id, file_hash);
-
-
---
--- Name: ingested_files ingested_files_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.ingested_files
-    ADD CONSTRAINT ingested_files_pkey PRIMARY KEY (id);
-
-
---
--- Name: ingestion_retry_logs ingestion_retry_logs_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.ingestion_retry_logs
-    ADD CONSTRAINT ingestion_retry_logs_pkey PRIMARY KEY (id);
-
-
---
--- Name: intent_feedback intent_feedback_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.intent_feedback
-    ADD CONSTRAINT intent_feedback_pkey PRIMARY KEY (id);
-
-
---
--- Name: rerank_feedback rerank_feedback_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.rerank_feedback
-    ADD CONSTRAINT rerank_feedback_pkey PRIMARY KEY (id);
-
-
---
--- Name: category_centroids_type_idx; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX category_centroids_type_idx ON public.category_centroids USING btree (document_type);
-
-
---
--- Name: category_centroids_user_id_idx; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX category_centroids_user_id_idx ON public.category_centroids USING btree (user_id);
-
-
---
--- Name: category_centroids_user_type_uidx; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE UNIQUE INDEX category_centroids_user_type_uidx ON public.category_centroids USING btree (user_id, document_type);
-
-
---
--- Name: chat_memory_user_id_idx; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX chat_memory_user_id_idx ON public.chat_memory USING btree (user_id);
-
-
---
--- Name: doc_node_type_idx; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX doc_node_type_idx ON public.documents USING btree (node_type);
-
-
---
--- Name: documents_content_fts_idx; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX documents_content_fts_idx ON public.documents USING gin (to_tsvector('english'::regconfig, content));
-
-
---
--- Name: documents_embedding_hnsw_idx; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX documents_embedding_hnsw_idx ON public.documents USING hnsw (((embedding)::extensions.halfvec(2048)) extensions.halfvec_cosine_ops) WITH (m='16', ef_construction='64');
-
-
---
--- Name: documents_metadata_filehash_idx; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX documents_metadata_filehash_idx ON public.documents USING btree (((metadata ->> 'file_hash'::text)));
-
-
---
--- Name: documents_metadata_idx; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX documents_metadata_idx ON public.documents USING gin (metadata);
-
-
---
--- Name: documents_user_id_idx; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX documents_user_id_idx ON public.documents USING btree (user_id);
-
-
---
--- Name: evaluation_logs_evaluated_at_idx; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX evaluation_logs_evaluated_at_idx ON public.evaluation_logs USING btree (evaluated_at DESC);
-
-
---
--- Name: evaluation_logs_run_label_idx; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX evaluation_logs_run_label_idx ON public.evaluation_logs USING btree (run_label);
-
-
---
--- Name: idx_chat_memory_session; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX idx_chat_memory_session ON public.chat_memory USING btree (session_id);
-
-
---
--- Name: idx_document_trees_json; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX idx_document_trees_json ON public.document_trees USING gin (tree_json);
-
-
---
--- Name: ingested_files_hash_idx; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX ingested_files_hash_idx ON public.ingested_files USING btree (file_hash);
-
-
---
--- Name: ingested_files_user_file_hash_uidx; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE UNIQUE INDEX ingested_files_user_file_hash_uidx ON public.ingested_files USING btree (user_id, file_hash);
-
-
---
--- Name: ingested_files_user_id_idx; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX ingested_files_user_id_idx ON public.ingested_files USING btree (user_id);
-
-
---
--- Name: ingestion_retry_logs_created_at_idx; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX ingestion_retry_logs_created_at_idx ON public.ingestion_retry_logs USING btree (created_at DESC);
-
-
---
--- Name: ingestion_retry_logs_user_id_idx; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX ingestion_retry_logs_user_id_idx ON public.ingestion_retry_logs USING btree (user_id);
-
-
---
--- Name: intent_feedback_user_id_idx; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX intent_feedback_user_id_idx ON public.intent_feedback USING btree (user_id);
-
-
---
--- Name: mv_document_types_idx; Type: INDEX; Schema: public; Owner: -
---
-
--- CREATE UNIQUE INDEX mv_document_types_idx ON public.mv_document_types USING btree (document_type);
-
-
---
--- Name: rerank_feedback_doc_type_idx; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX rerank_feedback_doc_type_idx ON public.rerank_feedback USING btree (document_type);
-
-
---
--- Name: rerank_feedback_user_created_idx; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX rerank_feedback_user_created_idx ON public.rerank_feedback USING btree (user_id, created_at DESC);
-
-
---
--- Name: category_centroids trg_centroids_updated_at; Type: TRIGGER; Schema: public; Owner: -
---
-
-CREATE TRIGGER trg_centroids_updated_at BEFORE UPDATE ON public.category_centroids FOR EACH ROW EXECUTE FUNCTION public._trg_set_updated_at();
-
-
---
--- Name: documents trg_refresh_mv_document_types; Type: TRIGGER; Schema: public; Owner: -
---
-
--- CREATE TRIGGER trg_refresh_mv_document_types AFTER INSERT ON public.documents FOR EACH STATEMENT EXECUTE FUNCTION public._trg_refresh_mv_document_types();
-
-
---
--- Name: category_centroids; Type: ROW SECURITY; Schema: public; Owner: -
---
-
-ALTER TABLE public.category_centroids ENABLE ROW LEVEL SECURITY;
-
---
--- Name: category_centroids centroids_delete_own; Type: POLICY; Schema: public; Owner: -
---
-
-CREATE POLICY centroids_delete_own ON public.category_centroids FOR DELETE USING ((user_id = auth.uid()));
-
-
---
--- Name: category_centroids centroids_insert_own; Type: POLICY; Schema: public; Owner: -
---
-
-CREATE POLICY centroids_insert_own ON public.category_centroids FOR INSERT WITH CHECK ((user_id = auth.uid()));
-
-
---
--- Name: category_centroids centroids_select_own; Type: POLICY; Schema: public; Owner: -
---
-
-CREATE POLICY centroids_select_own ON public.category_centroids FOR SELECT USING ((user_id = auth.uid()));
-
-
---
--- Name: category_centroids centroids_update_own; Type: POLICY; Schema: public; Owner: -
---
-
-CREATE POLICY centroids_update_own ON public.category_centroids FOR UPDATE USING ((user_id = auth.uid())) WITH CHECK ((user_id = auth.uid()));
-
-
---
--- Name: chat_memory; Type: ROW SECURITY; Schema: public; Owner: -
---
-
-ALTER TABLE public.chat_memory ENABLE ROW LEVEL SECURITY;
-
---
--- Name: chat_memory chat_memory_delete_own; Type: POLICY; Schema: public; Owner: -
---
-
-CREATE POLICY chat_memory_delete_own ON public.chat_memory FOR DELETE USING ((user_id = auth.uid()));
-
-
---
--- Name: chat_memory chat_memory_insert_own; Type: POLICY; Schema: public; Owner: -
---
-
-CREATE POLICY chat_memory_insert_own ON public.chat_memory FOR INSERT WITH CHECK ((user_id = auth.uid()));
-
-
---
--- Name: chat_memory chat_memory_select_own; Type: POLICY; Schema: public; Owner: -
---
-
-CREATE POLICY chat_memory_select_own ON public.chat_memory FOR SELECT USING ((user_id = auth.uid()));
-
-
---
--- Name: chat_memory chat_memory_update_own; Type: POLICY; Schema: public; Owner: -
---
-
-CREATE POLICY chat_memory_update_own ON public.chat_memory FOR UPDATE USING ((user_id = auth.uid())) WITH CHECK ((user_id = auth.uid()));
-
-
---
--- Name: documents; Type: ROW SECURITY; Schema: public; Owner: -
---
-
-ALTER TABLE public.documents ENABLE ROW LEVEL SECURITY;
+ALTER TABLE public.evaluation_datasets ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY (
+    SEQUENCE NAME public.evaluation_datasets_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1
+);
+
 
 --
--- Name: documents documents_delete_own; Type: POLICY; Schema: public; Owner: -
---
-
-CREATE POLICY documents_delete_own ON public.documents FOR DELETE USING ((user_id = auth.uid()));
-
-
---
--- Name: documents documents_insert_own; Type: POLICY; Schema: public; Owner: -
---
-
-CREATE POLICY documents_insert_own ON public.documents FOR INSERT WITH CHECK ((user_id = auth.uid()));
-
-
---
--- Name: documents documents_select_own; Type: POLICY; Schema: public; Owner: -
---
-
-CREATE POLICY documents_select_own ON public.documents FOR SELECT USING ((user_id = auth.uid()));
-
-
---
--- Name: documents documents_update_own; Type: POLICY; Schema: public; Owner: -
---
-
-CREATE POLICY documents_update_own ON public.documents FOR UPDATE USING ((user_id = auth.uid())) WITH CHECK ((user_id = auth.uid()));
+-- Name: evaluation_logs; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.evaluation_logs (
+    id uuid DEFAULT gen_random_uuid() NOT NULL,
+    run_label text,
+    evaluated_at timestamp with time zone,
+    alpha double precision,
+    k integer,
+    question text,
+    is_answerable boolean,
+    precision_at_k double precision,
+    faithfulness_proxy double precision,
+    relevance_proxy double precision,
+    local_reward double precision,
+    llm_judge_score double precision,
+    judge_a_verdict boolean,
+    judge_b_verdict boolean,
+    judge_a_model text,
+    judge_b_model text,
+    calibration_score double precision,
+    final_score double precision,
+    requires_manual_review boolean DEFAULT false,
+    disagreement_note text DEFAULT ''::text,
+    user_id uuid
+);
 
 
 --
--- Name: document_trees; Type: ROW SECURITY; Schema: public; Owner: -
+-- Name: graph_edges; Type: TABLE; Schema: public; Owner: -
 --
 
-ALTER TABLE public.document_trees ENABLE ROW LEVEL SECURITY;
+CREATE TABLE public.graph_edges (
+    id bigint NOT NULL,
+    user_id uuid,
+    source_node_key text NOT NULL,
+    target_node_key text NOT NULL,
+    edge_type text NOT NULL,
+    weight double precision DEFAULT 1.0 NOT NULL,
+    payload jsonb DEFAULT '{}'::jsonb NOT NULL,
+    created_at timestamp with time zone DEFAULT timezone('utc'::text, now()) NOT NULL
+);
+
 
 --
--- Name: document_trees document_trees_delete_own; Type: POLICY; Schema: public; Owner: -
+-- Name: graph_edges_id_seq; Type: SEQUENCE; Schema: public; Owner: -
 --
 
-CREATE POLICY document_trees_delete_own ON public.document_trees FOR DELETE USING ((user_id = auth.uid()));
+ALTER TABLE public.graph_edges ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY (
+    SEQUENCE NAME public.graph_edges_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1
+);
 
 
 --
--- Name: document_trees document_trees_insert_own; Type: POLICY; Schema: public; Owner: -
+-- Name: graph_nodes; Type: TABLE; Schema: public; Owner: -
 --
 
-CREATE POLICY document_trees_insert_own ON public.document_trees FOR INSERT WITH CHECK ((user_id = auth.uid()));
+CREATE TABLE public.graph_nodes (
+    id bigint NOT NULL,
+    user_id uuid,
+    node_key text NOT NULL,
+    node_type text NOT NULL,
+    label text NOT NULL,
+    payload jsonb DEFAULT '{}'::jsonb NOT NULL,
+    created_at timestamp with time zone DEFAULT timezone('utc'::text, now()) NOT NULL
+);
 
 
 --
--- Name: document_trees document_trees_select_own; Type: POLICY; Schema: public; Owner: -
+-- Name: graph_nodes_id_seq; Type: SEQUENCE; Schema: public; Owner: -
 --
 
-CREATE POLICY document_trees_select_own ON public.document_trees FOR SELECT USING ((user_id = auth.uid()));
+ALTER TABLE public.graph_nodes ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY (
+    SEQUENCE NAME public.graph_nodes_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1
+);
 
 
 --
--- Name: document_trees document_trees_update_own; Type: POLICY; Schema: public; Owner: -
+-- Name: ingested_files; Type: TABLE; Schema: public; Owner: -
 --
 
-CREATE POLICY document_trees_update_own ON public.document_trees FOR UPDATE USING ((user_id = auth.uid())) WITH CHECK ((user_id = auth.uid()));
+CREATE TABLE public.ingested_files (
+    id uuid DEFAULT gen_random_uuid() NOT NULL,
+    file_hash text NOT NULL,
+    filename text NOT NULL,
+    document_type text,
+    chunk_count integer DEFAULT 0,
+    ingested_at timestamp with time zone DEFAULT now(),
+    user_id uuid DEFAULT auth.uid(),
+    user_overridden boolean DEFAULT false,
+    identity_json jsonb DEFAULT '{}'::jsonb NOT NULL
+);
 
 
 --
--- Name: evaluation_logs; Type: ROW SECURITY; Schema: public; Owner: -
+-- Name: ingestion_retry_logs; Type: TABLE; Schema: public; Owner: -
 --
 
-ALTER TABLE public.evaluation_logs ENABLE ROW LEVEL SECURITY;
-
---
--- Name: evaluation_logs evaluation_logs_insert_own; Type: POLICY; Schema: public; Owner: -
---
-
-CREATE POLICY evaluation_logs_insert_own ON public.evaluation_logs FOR INSERT WITH CHECK ((user_id = auth.uid()));
-
-
---
--- Name: evaluation_logs evaluation_logs_select_own; Type: POLICY; Schema: public; Owner: -
---
-
-CREATE POLICY evaluation_logs_select_own ON public.evaluation_logs FOR SELECT USING ((user_id = auth.uid()));
+CREATE TABLE public.ingestion_retry_logs (
+    id bigint NOT NULL,
+    created_at timestamp with time zone DEFAULT now() NOT NULL,
+    user_id uuid,
+    file_hash text,
+    batch_num integer NOT NULL,
+    total_batches integer NOT NULL,
+    attempt integer NOT NULL,
+    event_type text NOT NULL,
+    message text,
+    sleep_s double precision DEFAULT 0
+);
 
 
 --
--- Name: ingestion_retry_logs; Type: ROW SECURITY; Schema: public; Owner: -
+-- Name: ingestion_retry_logs_id_seq; Type: SEQUENCE; Schema: public; Owner: -
 --
 
-ALTER TABLE public.ingestion_retry_logs ENABLE ROW LEVEL SECURITY;
+CREATE SEQUENCE public.ingestion_retry_logs_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
 
 --
--- Name: ingestion_retry_logs ingestion_retry_logs_delete_own; Type: POLICY; Schema: public; Owner: -
+-- Name: ingestion_retry_logs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
 --
 
-CREATE POLICY ingestion_retry_logs_delete_own ON public.ingestion_retry_logs FOR DELETE USING ((user_id = auth.uid()));
+ALTER SEQUENCE public.ingestion_retry_logs_id_seq OWNED BY public.ingestion_retry_logs.id;
 
 
 --
--- Name: ingestion_retry_logs ingestion_retry_logs_insert_own; Type: POLICY; Schema: public; Owner: -
+-- Name: intent_feedback; Type: TABLE; Schema: public; Owner: -
 --
 
-CREATE POLICY ingestion_retry_logs_insert_own ON public.ingestion_retry_logs FOR INSERT WITH CHECK ((user_id = auth.uid()));
+CREATE TABLE public.intent_feedback (
+    id bigint NOT NULL,
+    user_id uuid,
+    query text NOT NULL,
+    has_category boolean DEFAULT false NOT NULL,
+    has_history boolean DEFAULT false NOT NULL,
+    label integer NOT NULL,
+    created_at timestamp with time zone DEFAULT now() NOT NULL,
+    CONSTRAINT intent_feedback_label_check CHECK ((label = ANY (ARRAY[0, 1])))
+);
 
 
 --
--- Name: ingestion_retry_logs ingestion_retry_logs_select_own; Type: POLICY; Schema: public; Owner: -
+-- Name: intent_feedback_id_seq; Type: SEQUENCE; Schema: public; Owner: -
 --
 
-CREATE POLICY ingestion_retry_logs_select_own ON public.ingestion_retry_logs FOR SELECT USING ((user_id = auth.uid()));
+CREATE SEQUENCE public.intent_feedback_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
 
 
 --
--- Name: ingestion_retry_logs ingestion_retry_logs_update_own; Type: POLICY; Schema: public; Owner: -
+-- Name: intent_feedback_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
 --
 
-CREATE POLICY ingestion_retry_logs_update_own ON public.ingestion_retry_logs FOR UPDATE USING ((user_id = auth.uid())) WITH CHECK ((user_id = auth.uid()));
+ALTER SEQUENCE public.intent_feedback_id_seq OWNED BY public.intent_feedback.id;
 
 
 --
--- Name: ingested_files; Type: ROW SECURITY; Schema: public; Owner: -
+-- Name: query_traces; Type: TABLE; Schema: public; Owner: -
 --
 
-ALTER TABLE public.ingested_files ENABLE ROW LEVEL SECURITY;
-
---
--- Name: ingested_files ingested_files_delete_own; Type: POLICY; Schema: public; Owner: -
---
-
-CREATE POLICY ingested_files_delete_own ON public.ingested_files FOR DELETE USING ((user_id = auth.uid()));
-
-
---
--- Name: ingested_files ingested_files_insert_own; Type: POLICY; Schema: public; Owner: -
---
-
-CREATE POLICY ingested_files_insert_own ON public.ingested_files FOR INSERT WITH CHECK ((user_id = auth.uid()));
-
-
---
--- Name: ingested_files ingested_files_select_own; Type: POLICY; Schema: public; Owner: -
---
-
-CREATE POLICY ingested_files_select_own ON public.ingested_files FOR SELECT USING ((user_id = auth.uid()));
-
-
---
--- Name: ingested_files ingested_files_update_own; Type: POLICY; Schema: public; Owner: -
---
-
-CREATE POLICY ingested_files_update_own ON public.ingested_files FOR UPDATE USING ((user_id = auth.uid())) WITH CHECK ((user_id = auth.uid()));
-
-
---
--- Name: intent_feedback; Type: ROW SECURITY; Schema: public; Owner: -
---
-
-ALTER TABLE public.intent_feedback ENABLE ROW LEVEL SECURITY;
-
---
--- Name: intent_feedback intent_feedback_insert_own; Type: POLICY; Schema: public; Owner: -
---
-
-CREATE POLICY intent_feedback_insert_own ON public.intent_feedback FOR INSERT WITH CHECK ((user_id = auth.uid()));
-
-
---
--- Name: intent_feedback intent_feedback_select_own; Type: POLICY; Schema: public; Owner: -
---
-
-CREATE POLICY intent_feedback_select_own ON public.intent_feedback FOR SELECT USING ((user_id = auth.uid()));
-
-
---
--- Name: rerank_feedback; Type: ROW SECURITY; Schema: public; Owner: -
---
-
-ALTER TABLE public.rerank_feedback ENABLE ROW LEVEL SECURITY;
-
---
--- Name: rerank_feedback rerank_feedback_select_own; Type: POLICY; Schema: public; Owner: -
---
-
-CREATE POLICY rerank_feedback_select_own ON public.rerank_feedback FOR SELECT USING ((user_id = auth.uid()));
-
-
+CREATE TABLE public.query_traces (
+    trace_id uuid DEFAULT gen_random_uuid() NOT NULL,
+    user_id uuid,
+    session_id text DEFAULT 'default_session'::text NOT NULL,
+    question text NOT NULL,
+    route_mode text DEFAULT 'default'::text NOT NULL,
+    selected_experts jsonb DEFAULT '[]'::jsonb NOT NULL,
+    expert_weights jsonb DEFAULT '{}'::jsonb NOT NULL,
+    pinned_file_hashes jsonb DEFAULT '[]'::jsonb NOT NULL,
+    candidate_counts jsonb DEFAULT '{}'::jsonb NOT NULL,
+    selected_chunk_ids jsonb DEFAULT '[]'::jsonb NOT NULL,
+    doc_diagnostics jsonb DEFAULT '[]'::jsonb NOT NULL,
+    failure_modes jsonb DEFAULT '[]'::jsonb NOT NULL,
+    quality_metrics jsonb DEFAULT '{}'::jsonb NOT NULL,
+    answer_hash text,
+    answer_preview text,
+    latency_ms integer,
+    created_at timestamp with time zone DEFAULT timezone('utc'::text, now()) NOT NULL,
+    review_state text DEFAULT 'pending'::text NOT NULL,
+    review_notes text,
+    reviewed_at timestamp with time zone,
+    reviewed_by text,
+    promoted_to_eval boolean DEFAULT false NOT NULL,
+    document_types jsonb DEFAULT '[]'::jsonb NOT NULL
+);
+
+
 --
--- PostgreSQL database dump complete
+-- Name: rerank_feedback; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.rerank_feedback (
+    id bigint NOT NULL,
+    user_id uuid,
+    query_hash text NOT NULL,
+    chunk_id uuid,
+    chunk_hash text NOT NULL,
+    document_type text,
+    cohere_score real NOT NULL,
+    was_selected boolean NOT NULL,
+    created_at timestamp with time zone DEFAULT now() NOT NULL,
+    query_text text,
+    chunk_text text
+);
+
+
+--
+-- Name: rerank_feedback_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.rerank_feedback_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: rerank_feedback_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.rerank_feedback_id_seq OWNED BY public.rerank_feedback.id;
+
+
+--
+-- Name: ingestion_retry_logs id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.ingestion_retry_logs ALTER COLUMN id SET DEFAULT nextval('public.ingestion_retry_logs_id_seq'::regclass);
+
+
+--
+-- Name: intent_feedback id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.intent_feedback ALTER COLUMN id SET DEFAULT nextval('public.intent_feedback_id_seq'::regclass);
+
+
+--
+-- Name: rerank_feedback id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.rerank_feedback ALTER COLUMN id SET DEFAULT nextval('public.rerank_feedback_id_seq'::regclass);
+
+
+--
+-- Name: answer_feedback answer_feedback_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.answer_feedback
+    ADD CONSTRAINT answer_feedback_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: category_centroids category_centroids_document_type_key; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.category_centroids
+    ADD CONSTRAINT category_centroids_document_type_key UNIQUE (document_type);
+
+
+--
+-- Name: category_centroids category_centroids_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.category_centroids
+    ADD CONSTRAINT category_centroids_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: chat_memory chat_memory_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.chat_memory
+    ADD CONSTRAINT chat_memory_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: document_trees document_trees_user_file_hash_key; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.document_trees
+    ADD CONSTRAINT document_trees_user_file_hash_key UNIQUE (user_id, file_hash);
+
+
+--
+-- Name: documents documents_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.documents
+    ADD CONSTRAINT documents_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: evaluation_datasets evaluation_datasets_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.evaluation_datasets
+    ADD CONSTRAINT evaluation_datasets_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: evaluation_datasets evaluation_datasets_trace_id_key; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.evaluation_datasets
+    ADD CONSTRAINT evaluation_datasets_trace_id_key UNIQUE (trace_id);
+
+
 --
-
+-- Name: evaluation_logs evaluation_logs_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.evaluation_logs
+    ADD CONSTRAINT evaluation_logs_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: graph_edges graph_edges_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.graph_edges
+    ADD CONSTRAINT graph_edges_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: graph_edges graph_edges_user_id_source_node_key_target_node_key_edge_ty_key; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.graph_edges
+    ADD CONSTRAINT graph_edges_user_id_source_node_key_target_node_key_edge_ty_key UNIQUE (user_id, source_node_key, target_node_key, edge_type);
+
+
+--
+-- Name: graph_nodes graph_nodes_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.graph_nodes
+    ADD CONSTRAINT graph_nodes_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: graph_nodes graph_nodes_user_id_node_key_key; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.graph_nodes
+    ADD CONSTRAINT graph_nodes_user_id_node_key_key UNIQUE (user_id, node_key);
+
+
+--
+-- Name: ingested_files ingested_files_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.ingested_files
+    ADD CONSTRAINT ingested_files_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: ingested_files ingested_files_user_file_hash_key; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.ingested_files
+    ADD CONSTRAINT ingested_files_user_file_hash_key UNIQUE (user_id, file_hash);
+
+
+--
+-- Name: ingestion_retry_logs ingestion_retry_logs_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.ingestion_retry_logs
+    ADD CONSTRAINT ingestion_retry_logs_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: intent_feedback intent_feedback_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.intent_feedback
+    ADD CONSTRAINT intent_feedback_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: query_traces query_traces_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.query_traces
+    ADD CONSTRAINT query_traces_pkey PRIMARY KEY (trace_id);
+
+
+--
+-- Name: rerank_feedback rerank_feedback_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.rerank_feedback
+    ADD CONSTRAINT rerank_feedback_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: category_centroids_type_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX category_centroids_type_idx ON public.category_centroids USING btree (document_type);
+
+
+--
+-- Name: category_centroids_user_id_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX category_centroids_user_id_idx ON public.category_centroids USING btree (user_id);
+
+
+--
+-- Name: category_centroids_user_type_uidx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE UNIQUE INDEX category_centroids_user_type_uidx ON public.category_centroids USING btree (user_id, document_type);
+
+
+--
+-- Name: chat_memory_user_id_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX chat_memory_user_id_idx ON public.chat_memory USING btree (user_id);
+
+
+--
+-- Name: doc_node_type_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX doc_node_type_idx ON public.documents USING btree (node_type);
+
+
+--
+-- Name: documents_content_fts_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX documents_content_fts_idx ON public.documents USING gin (to_tsvector('english'::regconfig, content));
+
+
+--
+-- Name: documents_embedding_hnsw_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX documents_embedding_hnsw_idx ON public.documents USING hnsw (((embedding)::extensions.halfvec(2048)) extensions.halfvec_cosine_ops) WITH (m='16', ef_construction='64');
+
+
+--
+-- Name: documents_metadata_filehash_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX documents_metadata_filehash_idx ON public.documents USING btree (((metadata ->> 'file_hash'::text)));
+
+
+--
+-- Name: documents_metadata_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX documents_metadata_idx ON public.documents USING gin (metadata);
+
+
+--
+-- Name: documents_user_id_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX documents_user_id_idx ON public.documents USING btree (user_id);
+
+
+--
+-- Name: evaluation_logs_evaluated_at_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX evaluation_logs_evaluated_at_idx ON public.evaluation_logs USING btree (evaluated_at DESC);
+
+
+--
+-- Name: evaluation_logs_run_label_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX evaluation_logs_run_label_idx ON public.evaluation_logs USING btree (run_label);
+
+
+--
+-- Name: idx_answer_feedback_review_state_created; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_answer_feedback_review_state_created ON public.answer_feedback USING btree (review_state, created_at DESC);
+
+
+--
+-- Name: idx_answer_feedback_trace_created; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_answer_feedback_trace_created ON public.answer_feedback USING btree (trace_id, created_at DESC);
+
+
+--
+-- Name: idx_answer_feedback_user_created; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_answer_feedback_user_created ON public.answer_feedback USING btree (user_id, created_at DESC);
+
+
+--
+-- Name: idx_chat_memory_session; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_chat_memory_session ON public.chat_memory USING btree (session_id);
+
+
+--
+-- Name: idx_document_trees_json; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_document_trees_json ON public.document_trees USING gin (tree_json);
+
+
+--
+-- Name: idx_evaluation_datasets_active_created; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_evaluation_datasets_active_created ON public.evaluation_datasets USING btree (is_active, created_at DESC);
+
+
+--
+-- Name: idx_graph_edges_user_source; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_graph_edges_user_source ON public.graph_edges USING btree (user_id, source_node_key);
+
+
+--
+-- Name: idx_graph_edges_user_target; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_graph_edges_user_target ON public.graph_edges USING btree (user_id, target_node_key);
+
+
+--
+-- Name: idx_graph_nodes_user_label; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_graph_nodes_user_label ON public.graph_nodes USING btree (user_id, label);
+
+
+--
+-- Name: idx_graph_nodes_user_type; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_graph_nodes_user_type ON public.graph_nodes USING btree (user_id, node_type);
+
+
+--
+-- Name: idx_query_traces_review_state_created; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_query_traces_review_state_created ON public.query_traces USING btree (review_state, created_at DESC);
+
+
+--
+-- Name: idx_query_traces_session_created; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_query_traces_session_created ON public.query_traces USING btree (session_id, created_at DESC);
+
+
+--
+-- Name: idx_query_traces_user_created; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_query_traces_user_created ON public.query_traces USING btree (user_id, created_at DESC);
+
+
+--
+-- Name: ingested_files_hash_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX ingested_files_hash_idx ON public.ingested_files USING btree (file_hash);
+
+
+--
+-- Name: ingested_files_user_file_hash_uidx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE UNIQUE INDEX ingested_files_user_file_hash_uidx ON public.ingested_files USING btree (user_id, file_hash);
+
+
+--
+-- Name: ingested_files_user_id_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX ingested_files_user_id_idx ON public.ingested_files USING btree (user_id);
+
+
+--
+-- Name: ingestion_retry_logs_created_at_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX ingestion_retry_logs_created_at_idx ON public.ingestion_retry_logs USING btree (created_at DESC);
+
+
+--
+-- Name: ingestion_retry_logs_user_file_event_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX ingestion_retry_logs_user_file_event_idx ON public.ingestion_retry_logs USING btree (user_id, file_hash, event_type, created_at DESC);
+
+
+--
+-- Name: ingestion_retry_logs_user_id_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX ingestion_retry_logs_user_id_idx ON public.ingestion_retry_logs USING btree (user_id);
+
+
+--
+-- Name: intent_feedback_user_id_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX intent_feedback_user_id_idx ON public.intent_feedback USING btree (user_id);
+
+
+--
+-- Name: rerank_feedback_doc_type_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX rerank_feedback_doc_type_idx ON public.rerank_feedback USING btree (document_type);
+
+
+--
+-- Name: rerank_feedback_user_created_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX rerank_feedback_user_created_idx ON public.rerank_feedback USING btree (user_id, created_at DESC);
+
+
+--
+-- Name: category_centroids trg_centroids_updated_at; Type: TRIGGER; Schema: public; Owner: -
+--
+
+CREATE TRIGGER trg_centroids_updated_at BEFORE UPDATE ON public.category_centroids FOR EACH ROW EXECUTE FUNCTION public._trg_set_updated_at();
+
+
+--
+-- Name: answer_feedback answer_feedback_trace_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.answer_feedback
+    ADD CONSTRAINT answer_feedback_trace_id_fkey FOREIGN KEY (trace_id) REFERENCES public.query_traces(trace_id) ON DELETE CASCADE;
+
+
+--
+-- Name: evaluation_datasets evaluation_datasets_trace_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.evaluation_datasets
+    ADD CONSTRAINT evaluation_datasets_trace_id_fkey FOREIGN KEY (trace_id) REFERENCES public.query_traces(trace_id) ON DELETE SET NULL;
+
+
+--
+-- Name: answer_feedback; Type: ROW SECURITY; Schema: public; Owner: -
+--
+
+ALTER TABLE public.answer_feedback ENABLE ROW LEVEL SECURITY;
+
+--
+-- Name: category_centroids; Type: ROW SECURITY; Schema: public; Owner: -
+--
+
+ALTER TABLE public.category_centroids ENABLE ROW LEVEL SECURITY;
+
+--
+-- Name: chat_memory; Type: ROW SECURITY; Schema: public; Owner: -
+--
+
+ALTER TABLE public.chat_memory ENABLE ROW LEVEL SECURITY;
+
+--
+-- Name: chat_memory chat_memory_delete_own; Type: POLICY; Schema: public; Owner: -
+--
+
+CREATE POLICY chat_memory_delete_own ON public.chat_memory FOR DELETE USING ((user_id = auth.uid()));
+
+
+--
+-- Name: chat_memory chat_memory_insert_own; Type: POLICY; Schema: public; Owner: -
+--
+
+CREATE POLICY chat_memory_insert_own ON public.chat_memory FOR INSERT WITH CHECK ((user_id = auth.uid()));
+
+
+--
+-- Name: chat_memory chat_memory_select_own; Type: POLICY; Schema: public; Owner: -
+--
+
+CREATE POLICY chat_memory_select_own ON public.chat_memory FOR SELECT USING ((user_id = auth.uid()));
+
+
+--
+-- Name: chat_memory chat_memory_update_own; Type: POLICY; Schema: public; Owner: -
+--
+
+CREATE POLICY chat_memory_update_own ON public.chat_memory FOR UPDATE USING ((user_id = auth.uid())) WITH CHECK ((user_id = auth.uid()));
+
+
+--
+-- Name: document_trees; Type: ROW SECURITY; Schema: public; Owner: -
+--
+
+ALTER TABLE public.document_trees ENABLE ROW LEVEL SECURITY;
+
+--
+-- Name: document_trees document_trees_delete_own; Type: POLICY; Schema: public; Owner: -
+--
+
+CREATE POLICY document_trees_delete_own ON public.document_trees FOR DELETE USING ((user_id = auth.uid()));
+
+
+--
+-- Name: document_trees document_trees_insert_own; Type: POLICY; Schema: public; Owner: -
+--
+
+CREATE POLICY document_trees_insert_own ON public.document_trees FOR INSERT WITH CHECK ((user_id = auth.uid()));
+
+
+--
+-- Name: document_trees document_trees_select_own; Type: POLICY; Schema: public; Owner: -
+--
+
+CREATE POLICY document_trees_select_own ON public.document_trees FOR SELECT USING ((user_id = auth.uid()));
+
+
+--
+-- Name: document_trees document_trees_update_own; Type: POLICY; Schema: public; Owner: -
+--
+
+CREATE POLICY document_trees_update_own ON public.document_trees FOR UPDATE USING ((user_id = auth.uid())) WITH CHECK ((user_id = auth.uid()));
+
+
+--
+-- Name: documents; Type: ROW SECURITY; Schema: public; Owner: -
+--
+
+ALTER TABLE public.documents ENABLE ROW LEVEL SECURITY;
+
+--
+-- Name: documents documents_delete_own; Type: POLICY; Schema: public; Owner: -
+--
+
+CREATE POLICY documents_delete_own ON public.documents FOR DELETE USING ((user_id = auth.uid()));
+
+
+--
+-- Name: documents documents_insert_own; Type: POLICY; Schema: public; Owner: -
+--
+
+CREATE POLICY documents_insert_own ON public.documents FOR INSERT WITH CHECK ((user_id = auth.uid()));
+
+
+--
+-- Name: documents documents_select_own; Type: POLICY; Schema: public; Owner: -
+--
+
+CREATE POLICY documents_select_own ON public.documents FOR SELECT USING ((user_id = auth.uid()));
+
+
+--
+-- Name: documents documents_update_own; Type: POLICY; Schema: public; Owner: -
+--
+
+CREATE POLICY documents_update_own ON public.documents FOR UPDATE USING ((user_id = auth.uid())) WITH CHECK ((user_id = auth.uid()));
+
+
+--
+-- Name: evaluation_datasets; Type: ROW SECURITY; Schema: public; Owner: -
+--
+
+ALTER TABLE public.evaluation_datasets ENABLE ROW LEVEL SECURITY;
+
+--
+-- Name: evaluation_logs; Type: ROW SECURITY; Schema: public; Owner: -
+--
+
+ALTER TABLE public.evaluation_logs ENABLE ROW LEVEL SECURITY;
+
+--
+-- Name: graph_edges; Type: ROW SECURITY; Schema: public; Owner: -
+--
+
+ALTER TABLE public.graph_edges ENABLE ROW LEVEL SECURITY;
+
+--
+-- Name: graph_nodes; Type: ROW SECURITY; Schema: public; Owner: -
+--
+
+ALTER TABLE public.graph_nodes ENABLE ROW LEVEL SECURITY;
+
+--
+-- Name: ingested_files; Type: ROW SECURITY; Schema: public; Owner: -
+--
+
+ALTER TABLE public.ingested_files ENABLE ROW LEVEL SECURITY;
+
+--
+-- Name: ingested_files ingested_files_delete_own; Type: POLICY; Schema: public; Owner: -
+--
+
+CREATE POLICY ingested_files_delete_own ON public.ingested_files FOR DELETE USING ((user_id = auth.uid()));
+
+
+--
+-- Name: ingested_files ingested_files_insert_own; Type: POLICY; Schema: public; Owner: -
+--
+
+CREATE POLICY ingested_files_insert_own ON public.ingested_files FOR INSERT WITH CHECK ((user_id = auth.uid()));
+
+
+--
+-- Name: ingested_files ingested_files_select_own; Type: POLICY; Schema: public; Owner: -
+--
+
+CREATE POLICY ingested_files_select_own ON public.ingested_files FOR SELECT USING ((user_id = auth.uid()));
+
+
+--
+-- Name: ingested_files ingested_files_update_own; Type: POLICY; Schema: public; Owner: -
+--
+
+CREATE POLICY ingested_files_update_own ON public.ingested_files FOR UPDATE USING ((user_id = auth.uid())) WITH CHECK ((user_id = auth.uid()));
+
+
+--
+-- Name: ingestion_retry_logs; Type: ROW SECURITY; Schema: public; Owner: -
+--
+
+ALTER TABLE public.ingestion_retry_logs ENABLE ROW LEVEL SECURITY;
+
+--
+-- Name: intent_feedback; Type: ROW SECURITY; Schema: public; Owner: -
+--
+
+ALTER TABLE public.intent_feedback ENABLE ROW LEVEL SECURITY;
+
+--
+-- Name: query_traces; Type: ROW SECURITY; Schema: public; Owner: -
+--
+
+ALTER TABLE public.query_traces ENABLE ROW LEVEL SECURITY;
+
+--
+-- Name: rerank_feedback; Type: ROW SECURITY; Schema: public; Owner: -
+--
+
+ALTER TABLE public.rerank_feedback ENABLE ROW LEVEL SECURITY;
+
+--
+-- PostgreSQL database dump complete
+--
+
+\unrestrict 32urOXpOnsQS0zoo7jGTkIs0BeRgGPyJVLWPDJ6IexS9GSsM4lpkxJaAg6FM0Ua

tests/test_guest_mode.py

@@ -0,0 +1,74 @@
+import jwt
+from starlette.requests import Request
+
+from backend.core.auth_utils import is_guest_token
+from backend.main import _rate_limit_key
+
+_TEST_GUEST_KEY = "guest-secret-key-that-is-long-enough"
+_TEST_USER_KEY = "user-secret-key-that-is-long-enough"
+
+
+def _make_request(headers: dict[str, str], client_ip: str = "127.0.0.1") -> Request:
+    scope = {
+        "type": "http",
+        "method": "GET",
+        "path": "/",
+        "scheme": "http",
+        "client": (client_ip, 4321),
+        "server": ("testserver", 80),
+        "headers": [
+            (key.lower().encode("latin-1"), value.encode("latin-1"))
+            for key, value in headers.items()
+        ],
+    }
+    return Request(scope)
+
+
+def test_is_guest_token_detects_anonymous_provider():
+    token = jwt.encode(
+        {
+            "sub": "11111111-1111-1111-1111-111111111111",
+            "app_metadata": {"provider": "anonymous", "providers": ["anonymous"]},
+        },
+        _TEST_GUEST_KEY,
+        algorithm="HS256",
+    )
+    assert is_guest_token(token) is True
+
+
+def test_is_guest_token_ignores_regular_authenticated_user():
+    token = jwt.encode(
+        {
+            "sub": "22222222-2222-2222-2222-222222222222",
+            "app_metadata": {"provider": "email", "providers": ["email"]},
+        },
+        _TEST_USER_KEY,
+        algorithm="HS256",
+    )
+    assert is_guest_token(token) is False
+
+
+def test_rate_limit_key_uses_ip_for_guest_tokens():
+    token = jwt.encode(
+        {
+            "sub": "33333333-3333-3333-3333-333333333333",
+            "app_metadata": {"provider": "anonymous"},
+        },
+        _TEST_GUEST_KEY,
+        algorithm="HS256",
+    )
+    request = _make_request({"X-Auth-Token": token}, client_ip="10.0.0.8")
+    assert _rate_limit_key(request) == "10.0.0.8"
+
+
+def test_rate_limit_key_uses_token_for_regular_users():
+    token = jwt.encode(
+        {
+            "sub": "44444444-4444-4444-4444-444444444444",
+            "app_metadata": {"provider": "email"},
+        },
+        _TEST_USER_KEY,
+        algorithm="HS256",
+    )
+    request = _make_request({"Authorization": f"Bearer {token}"}, client_ip="10.0.0.8")
+    assert _rate_limit_key(request) == token

tests/test_ingest_api.py

@@ -0,0 +1,156 @@
+import asyncio
+import os
+import sys
+import tempfile
+from types import SimpleNamespace
+
+import pytest
+from fastapi import HTTPException
+from starlette.requests import Request
+
+from backend.api import ingest as ingest_api
+from backend.core import pipeline, tasks
+
+
+class FakeUploadFile:
+    def __init__(self, filename: str, content: bytes):
+        self.filename = filename
+        self._content = content
+        self._cursor = 0
+
+    async def read(self, size: int = -1) -> bytes:
+        if size is None or size < 0:
+            size = len(self._content) - self._cursor
+        start = self._cursor
+        end = min(len(self._content), self._cursor + size)
+        self._cursor = end
+        return self._content[start:end]
+
+    async def seek(self, offset: int) -> None:
+        self._cursor = max(0, offset)
+
+
+class FakeCountQuery:
+    def __init__(self, count: int):
+        self.count = count
+
+    def select(self, *_args, **_kwargs):
+        return self
+
+    def eq(self, *_args, **_kwargs):
+        return self
+
+    def execute(self):
+        return SimpleNamespace(count=self.count)
+
+
+class FakeCountSupabase:
+    def __init__(self, count: int = 0):
+        self.count = count
+
+    def table(self, _name: str):
+        return FakeCountQuery(self.count)
+
+
+def _install_fake_magic(monkeypatch):
+    monkeypatch.setitem(
+        sys.modules,
+        "magic",
+        SimpleNamespace(from_buffer=lambda *_args, **_kwargs: "application/pdf"),
+    )
+
+
+def _fake_request() -> Request:
+    return Request(
+        {
+            "type": "http",
+            "method": "POST",
+            "path": "/api/v1/ingest/upload",
+            "headers": [],
+            "client": ("127.0.0.1", 12345),
+        }
+    )
+
+
+def test_upload_rejects_large_pdf_with_original_http_status(monkeypatch):
+    _install_fake_magic(monkeypatch)
+    monkeypatch.setattr(
+        pipeline,
+        "_build_supabase_client",
+        lambda **_kwargs: FakeCountSupabase(count=0),
+    )
+    monkeypatch.setattr(ingest_api, "celery_app", SimpleNamespace())
+    monkeypatch.setattr(
+        ingest_api,
+        "process_pdf_task",
+        SimpleNamespace(delay=lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("should not queue"))),
+    )
+    monkeypatch.setattr(ingest_api.config, "MAX_UPLOAD_MB", 1, raising=False)
+    monkeypatch.setattr(ingest_api.config, "GUEST_MAX_UPLOAD_MB", 1, raising=False)
+
+    file = FakeUploadFile("guide.pdf", b"%PDF-1.4\n" + (b"x" * (2 * 1024 * 1024)))
+
+    with pytest.raises(HTTPException) as exc_info:
+        asyncio.run(
+            ingest_api.upload(
+                request=_fake_request(),
+                file=file,
+                user_id="user-1",
+                x_auth_token="token",
+            )
+        )
+
+    assert exc_info.value.status_code == 413
+
+
+def test_upload_returns_503_when_worker_is_unavailable(monkeypatch):
+    _install_fake_magic(monkeypatch)
+    monkeypatch.setattr(
+        pipeline,
+        "_build_supabase_client",
+        lambda **_kwargs: FakeCountSupabase(count=0),
+    )
+    monkeypatch.setattr(ingest_api, "celery_app", None)
+    monkeypatch.setattr(ingest_api, "process_pdf_task", SimpleNamespace())
+
+    file = FakeUploadFile("guide.pdf", b"%PDF-1.4\nsmall")
+
+    with pytest.raises(HTTPException) as exc_info:
+        asyncio.run(
+            ingest_api.upload(
+                request=_fake_request(),
+                file=file,
+                user_id="user-1",
+                x_auth_token="token",
+            )
+        )
+
+    assert exc_info.value.status_code == 503
+    assert "worker is unavailable" in exc_info.value.detail.lower()
+
+
+def test_get_ingest_status_requires_available_worker(monkeypatch):
+    monkeypatch.setattr(ingest_api, "celery_app", None)
+
+    with pytest.raises(HTTPException) as exc_info:
+        ingest_api.get_ingest_status("task-1")
+
+    assert exc_info.value.status_code == 503
+
+
+def test_process_pdf_task_impl_preserves_original_exception_and_cleans_temp_file(monkeypatch):
+    fd, tmp_path = tempfile.mkstemp(suffix="_guide.pdf")
+    os.close(fd)
+
+    monkeypatch.setattr(
+        tasks,
+        "run_ingestion",
+        lambda **_kwargs: (_ for _ in ()).throw(ValueError("boom")),
+    )
+
+    fake_task = SimpleNamespace(update_state=lambda **_kwargs: None)
+
+    with pytest.raises(ValueError, match="boom"):
+        tasks._process_pdf_task_impl(fake_task, tmp_path, "guide.pdf", "token")
+
+    assert not os.path.exists(tmp_path)

tests/test_pipeline_regressions.py

@@ -0,0 +1,1831 @@
+import asyncio
+import json
+import os
+from types import SimpleNamespace
+
+from langchain_core.documents import Document
+
+from backend.api import admin
+from backend.api import query as query_api
+from backend.core import auth_utils, pipeline, providers
+from backend.eval import run_eval
+
+
+class FakeElement:
+    def __init__(self, text: str, category: str = "Text", page_number: int = 1):
+        self.text = text
+        self.category = category
+        self.metadata = SimpleNamespace(page_number=page_number)
+
+    def __str__(self) -> str:
+        return self.text
+
+
+class FakeIngestionTable:
+    def __init__(self, supabase, name: str):
+        self.supabase = supabase
+        self.name = name
+        self.action = None
+        self.filters = {}
+        self.payload = None
+
+    def select(self, *_args):
+        self.action = "select"
+        return self
+
+    def delete(self):
+        self.action = "delete"
+        return self
+
+    def upsert(self, payload, on_conflict=None):
+        self.action = "upsert"
+        self.payload = payload
+        self.on_conflict = on_conflict
+        return self
+
+    def insert(self, payload):
+        self.action = "insert"
+        self.payload = payload
+        return self
+
+    def eq(self, key, value):
+        self.filters[key] = value
+        return self
+
+    def contains(self, key, value):
+        self.filters[key] = value
+        return self
+
+    def limit(self, value):
+        self.filters["limit"] = value
+        return self
+
+    def execute(self):
+        self.supabase.ops.append((self.name, self.action, dict(self.filters)))
+        if self.action == "insert":
+            self.supabase.inserts.append((self.name, self.payload))
+        if self.name == "ingested_files" and self.action == "select":
+            return SimpleNamespace(
+                data=[{"document_type": "short_story", "user_overridden": True}]
+            )
+        return SimpleNamespace(data=[])
+
+
+class FakeIngestionSupabase:
+    def __init__(self):
+        self.ops = []
+        self.inserts = []
+
+    def table(self, name: str):
+        return FakeIngestionTable(self, name)
+
+
+class FakeRecoveryTable:
+    def __init__(self, supabase, name: str):
+        self.supabase = supabase
+        self.name = name
+        self.action = None
+        self.filters = {}
+        self.payload = None
+        self.limit_value = None
+
+    def select(self, *_args):
+        self.action = "select"
+        return self
+
+    def upsert(self, payload, on_conflict=None):
+        self.action = "upsert"
+        self.payload = payload
+        self.on_conflict = on_conflict
+        self.supabase.upserts.append((self.name, payload, on_conflict))
+        return self
+
+    def insert(self, payload):
+        self.action = "insert"
+        self.payload = payload
+        self.supabase.inserts.append((self.name, payload))
+        return self
+
+    def eq(self, key, value):
+        self.filters[key] = value
+        return self
+
+    def contains(self, key, value):
+        self.filters[key] = value
+        return self
+
+    def limit(self, value):
+        self.limit_value = value
+        return self
+
+    def execute(self):
+        if self.name == "documents" and self.action == "select":
+            file_hash = (self.filters.get("metadata") or {}).get("file_hash")
+            user_id = self.filters.get("user_id")
+            rows = [
+                row for row in self.supabase.documents
+                if (not user_id or row.get("user_id") == user_id)
+                and ((row.get("metadata") or {}).get("file_hash") == file_hash)
+            ]
+            if self.limit_value is not None:
+                rows = rows[: self.limit_value]
+            return SimpleNamespace(data=rows)
+        if self.name == "ingestion_retry_logs" and self.action == "select":
+            user_id = self.filters.get("user_id")
+            file_hash = self.filters.get("file_hash")
+            event_type = self.filters.get("event_type")
+            rows = [
+                row for row in self.supabase.ingestion_logs
+                if (not user_id or row.get("user_id") == user_id)
+                and (not file_hash or row.get("file_hash") == file_hash)
+                and (not event_type or row.get("event_type") == event_type)
+            ]
+            if self.limit_value is not None:
+                rows = rows[: self.limit_value]
+            return SimpleNamespace(data=rows)
+        return SimpleNamespace(data=[])
+
+
+class FakeRecoverySupabase:
+    def __init__(self, *, documents=None, ingestion_logs=None):
+        self.documents = list(documents or [])
+        self.ingestion_logs = list(ingestion_logs or [])
+        self.upserts = []
+        self.inserts = []
+
+    def table(self, name: str):
+        return FakeRecoveryTable(self, name)
+
+
+class FakeRetrieveTable:
+    def __init__(self, supabase, name: str):
+        self.supabase = supabase
+        self.name = name
+        self.filters = {}
+
+    def select(self, *_args):
+        return self
+
+    def in_(self, key, values):
+        self.filters[key] = tuple(values)
+        return self
+
+    def eq(self, key, value):
+        self.filters[key] = value
+        return self
+
+    def execute(self):
+        if self.name == "ingested_files":
+            requested = self.filters.get("file_hash", ())
+            data = []
+            for item in requested:
+                if item == "A":
+                    data.append({"file_hash": "A", "filename": "About Love Anton Chekhov"})
+                if item == "B":
+                    data.append({"file_hash": "B", "filename": "BEYOND BOUNDS"})
+            return SimpleNamespace(data=data)
+        return SimpleNamespace(data=[])
+
+
+class FakeRetrieveRpc:
+    def __init__(self, supabase, params):
+        self.supabase = supabase
+        self.params = params
+
+    def execute(self):
+        file_hash = self.params["filter"]["file_hash"]
+        if file_hash == "A":
+            return SimpleNamespace(
+                data=[
+                    {
+                        "id": "A-1",
+                        "content": "A" * 400,
+                        "metadata": {
+                            "file_hash": "A",
+                            "source": "About Love Anton Chekhov",
+                            "chunk_index": 1,
+                            "document_type": "short_story",
+                            "page_numbers": [1],
+                        },
+                    },
+                    {
+                        "id": "A-2",
+                        "content": "B" * 400,
+                        "metadata": {
+                            "file_hash": "A",
+                            "source": "About Love Anton Chekhov",
+                            "chunk_index": 2,
+                            "document_type": "short_story",
+                            "page_numbers": [2],
+                        },
+                    },
+                ]
+            )
+        return SimpleNamespace(
+            data=[
+                {
+                    "id": "B-1",
+                    "content": "C" * 200,
+                    "metadata": {
+                        "file_hash": "B",
+                        "source": "BEYOND BOUNDS",
+                        "chunk_index": 1,
+                        "document_type": "short_story",
+                        "page_numbers": [1],
+                    },
+                }
+            ]
+        )
+
+
+class FakeRetrieveSupabase:
+    def table(self, name: str):
+        return FakeRetrieveTable(self, name)
+
+    def rpc(self, _name: str, params):
+        return FakeRetrieveRpc(self, params)
+
+
+class FakeServiceTable:
+    def __init__(self, supabase, name: str):
+        self.supabase = supabase
+        self.name = name
+        self.filters = {}
+        self.action = None
+        self.payload = None
+
+    def insert(self, payload):
+        self.action = "insert"
+        self.payload = payload
+        self.supabase.inserts.append((self.name, payload))
+        return self
+
+    def update(self, payload):
+        self.action = "update"
+        self.payload = payload
+        return self
+
+    def upsert(self, payload, on_conflict=None):
+        self.action = "upsert"
+        self.payload = payload
+        self.on_conflict = on_conflict
+        self.supabase.upserts.append((self.name, payload, on_conflict))
+        return self
+
+    def select(self, *_args):
+        self.action = "select"
+        return self
+
+    def eq(self, key, value):
+        self.filters[key] = value
+        return self
+
+    def in_(self, key, values):
+        self.filters[key] = tuple(values)
+        return self
+
+    def limit(self, value):
+        self.filters["limit"] = value
+        return self
+
+    def execute(self):
+        if self.name == "query_traces" and self.action == "select":
+            trace_ids = self.filters.get("trace_id")
+            data = [
+                row
+                for row in self.supabase.trace_rows
+                if trace_ids is None
+                or (
+                    isinstance(trace_ids, tuple)
+                    and row.get("trace_id") in trace_ids
+                )
+                or row.get("trace_id") == trace_ids
+            ]
+            if "user_id" in self.filters:
+                data = [row for row in data if row.get("user_id") == self.filters["user_id"]]
+            if "session_id" in self.filters:
+                data = [row for row in data if row.get("session_id") == self.filters["session_id"]]
+            return SimpleNamespace(data=data[: self.filters.get("limit", len(data))])
+        if self.name == "answer_feedback" and self.action == "select":
+            rows = [
+                row
+                for row in self.supabase.feedback_rows
+                if ("promote_to_eval" not in self.filters or row.get("promote_to_eval") is self.filters["promote_to_eval"])
+            ]
+            if "user_id" in self.filters:
+                rows = [row for row in rows if row.get("user_id") == self.filters["user_id"]]
+            if "trace_id" in self.filters:
+                rows = [row for row in rows if row.get("trace_id") == self.filters["trace_id"]]
+            if "id" in self.filters:
+                rows = [row for row in rows if row.get("id") == self.filters["id"]]
+            return SimpleNamespace(data=rows[: self.filters.get("limit", len(rows))])
+        if self.name == "evaluation_datasets" and self.action == "select":
+            rows = list(self.supabase.eval_rows)
+            return SimpleNamespace(data=rows[: self.filters.get("limit", len(rows))])
+        if self.name == "query_traces" and self.action == "insert":
+            self.supabase.trace_rows.append(self.payload)
+        if self.name == "answer_feedback" and self.action == "insert":
+            self.supabase.feedback_rows.append(self.payload)
+        if self.name == "query_traces" and self.action == "update":
+            for row in self.supabase.trace_rows:
+                if all(row.get(k) == v for k, v in self.filters.items()):
+                    row.update(self.payload)
+        if self.name == "answer_feedback" and self.action == "update":
+            for row in self.supabase.feedback_rows:
+                if all(row.get(k) == v for k, v in self.filters.items()):
+                    row.update(self.payload)
+        if self.name == "evaluation_datasets" and self.action == "upsert":
+            trace_id = self.payload.get("trace_id")
+            existing = next(
+                (row for row in self.supabase.eval_rows if row.get("trace_id") == trace_id),
+                None,
+            )
+            if existing:
+                existing.update(self.payload)
+            else:
+                self.supabase.eval_rows.append(self.payload)
+        return SimpleNamespace(data=[])
+
+
+class FakeServiceSupabase:
+    def __init__(self):
+        self.inserts = []
+        self.upserts = []
+        self.trace_rows = []
+        self.feedback_rows = []
+        self.eval_rows = []
+
+    def table(self, name: str):
+        return FakeServiceTable(self, name)
+
+    def rpc(self, _name: str, _params):
+        return SimpleNamespace(execute=lambda: SimpleNamespace(data=[]))
+
+
+class FakeGraphServiceTable(FakeServiceTable):
+    def execute(self):
+        if self.name == "graph_nodes" and self.action == "select":
+            rows = list(self.supabase.graph_nodes)
+            if "user_id" in self.filters:
+                rows = [row for row in rows if row.get("user_id") == self.filters["user_id"]]
+            return SimpleNamespace(data=rows)
+        if self.name == "graph_edges" and self.action == "select":
+            rows = list(self.supabase.graph_edges)
+            if "user_id" in self.filters:
+                rows = [row for row in rows if row.get("user_id") == self.filters["user_id"]]
+            return SimpleNamespace(data=rows)
+        return super().execute()
+
+
+class FakeGraphServiceSupabase(FakeServiceSupabase):
+    def __init__(self):
+        super().__init__()
+        self.graph_nodes = []
+        self.graph_edges = []
+
+    def table(self, name: str):
+        return FakeGraphServiceTable(self, name)
+
+
+class FakeGraphVectorTable:
+    def __init__(self, rows):
+        self.rows = rows
+        self.filters = {}
+
+    def select(self, *_args):
+        return self
+
+    def eq(self, key, value):
+        self.filters[key] = value
+        return self
+
+    def contains(self, key, value):
+        self.filters[key] = value
+        return self
+
+    def execute(self):
+        rows = list(self.rows)
+        if "user_id" in self.filters:
+            rows = [row for row in rows if row.get("user_id") == self.filters["user_id"]]
+        metadata_contains = self.filters.get("metadata")
+        if metadata_contains:
+            rows = [
+                row for row in rows
+                if all((row.get("metadata", {}) or {}).get(k) == v for k, v in metadata_contains.items())
+            ]
+        return SimpleNamespace(data=rows)
+
+
+class FakeGraphVectorSupabase:
+    def __init__(self, rows):
+        self.rows = rows
+
+    def table(self, _name: str):
+        return FakeGraphVectorTable(self.rows)
+
+
+class FakeRerankResult:
+    def __init__(self, index: int, relevance_score: float):
+        self.index = index
+        self.relevance_score = relevance_score
+
+
+class FakeCohereClient:
+    def __init__(self, *_args, **_kwargs):
+        pass
+
+    def rerank(self, model, query, documents, top_n):
+        del model, query, top_n
+        if len(documents) == 2:
+            scores = [0.9, 0.8]
+        else:
+            scores = [0.2]
+        return SimpleNamespace(
+            results=[
+                FakeRerankResult(index=i, relevance_score=score)
+                for i, score in enumerate(scores[: len(documents)])
+            ]
+        )
+
+
+def test_create_chunks_uses_short_document_settings(monkeypatch):
+    seen = {}
+
+    def fake_chunk_by_title(elements, **kwargs):
+        seen["kwargs"] = kwargs
+        return list(elements)
+
+    monkeypatch.setattr(pipeline, "chunk_by_title", fake_chunk_by_title)
+
+    chunks = pipeline.create_chunks([FakeElement("short text")], text_chars=5_000)
+
+    assert len(chunks) == 1
+    assert seen["kwargs"] == {
+        "max_characters": 3000,
+        "new_after_n_chars": 2500,
+        "combine_text_under_n_chars": 300,
+    }
+
+
+def test_create_chunks_keeps_large_document_settings(monkeypatch):
+    seen = {}
+
+    def fake_chunk_by_title(elements, **kwargs):
+        seen["kwargs"] = kwargs
+        return list(elements)
+
+    monkeypatch.setattr(pipeline, "chunk_by_title", fake_chunk_by_title)
+
+    chunks = pipeline.create_chunks([FakeElement("large text")], text_chars=40_000)
+
+    assert len(chunks) == 1
+    assert seen["kwargs"] == {
+        "max_characters": 8000,
+        "new_after_n_chars": 7000,
+        "combine_text_under_n_chars": 500,
+    }
+
+
+def test_predict_and_prefetch_uses_rewriter_provider(monkeypatch):
+    seen_purposes = []
+    warmed_queries = []
+
+    class FakeLLM:
+        def invoke(self, _messages):
+            return SimpleNamespace(content='["follow-up question"]')
+
+    def fake_build_chat_llm(*, purpose="text", **_kwargs):
+        seen_purposes.append(purpose)
+        return FakeLLM()
+
+    monkeypatch.setattr(
+        providers.ProviderFactory, "build_chat_llm", staticmethod(fake_build_chat_llm)
+    )
+    monkeypatch.setattr(
+        pipeline,
+        "retrieve_chunks",
+        lambda **kwargs: warmed_queries.append(kwargs["query"]) or [],
+    )
+
+    pipeline._predict_and_prefetch(
+        original_query="original",
+        answer="answer",
+        category="short_story",
+        session_id="session-1",
+        access_token="token",
+    )
+
+    assert seen_purposes == ["rewriter"]
+    assert warmed_queries == ["follow-up question"]
+
+
+def test_generate_answer_stream_marks_summary_nodes(monkeypatch):
+    captured = {}
+
+    class FakeLLM:
+        async def astream(self, messages):
+            captured["prompt"] = messages[0].content[0]["text"]
+            yield SimpleNamespace(content="ok")
+
+    monkeypatch.setattr(pipeline, "_build_llm", lambda needs_vision=False: FakeLLM())
+    monkeypatch.setattr(pipeline, "_get_episodic_memory", lambda *args, **kwargs: "")
+    monkeypatch.setattr(pipeline, "_log_retrieval_reward", lambda *args, **kwargs: None)
+    monkeypatch.setattr(pipeline, "_save_to_memory", lambda *args, **kwargs: None)
+
+    summary_chunk = Document(
+        page_content="Summary body",
+        metadata={
+            "source": "About Love Anton Chekhov",
+            "node_type": "summary",
+            "node_level": 2,
+            "chunk_index": "summary-1",
+            "document_type": "short_story",
+            "relevance_score": 0.8,
+        },
+    )
+    leaf_chunk = Document(
+        page_content="Leaf fallback",
+        metadata={
+            "source": "About Love Anton Chekhov",
+            "chunk_index": 1,
+            "document_type": "short_story",
+            "relevance_score": 0.6,
+            "original_content": {"raw_text": "Leaf raw text", "tables_html": []},
+        },
+    )
+
+    async def collect():
+        events = []
+        async for event in pipeline.generate_answer_stream(
+            chunks=[summary_chunk, leaf_chunk],
+            query="summarise this",
+            access_token=None,
+            category="short_story",
+            priority_file_hashes=None,
+        ):
+            events.append(event)
+        return events
+
+    events = asyncio.run(collect())
+
+    assert any(event["type"] == "done" for event in events)
+    assert "[SYNTHESIZED CHAPTER SUMMARY - LEVEL 2]" in captured["prompt"]
+    assert "TEXT:\nSummary body" in captured["prompt"]
+    assert "TEXT:\nLeaf raw text" in captured["prompt"]
+
+
+def test_run_ingestion_preserves_user_override_before_cleanup(monkeypatch):
+    fake_supabase = FakeIngestionSupabase()
+    captured = {}
+
+    monkeypatch.setattr(auth_utils, "extract_jwt_sub", lambda _token: "user-1")
+    monkeypatch.setattr(pipeline, "get_file_fingerprint", lambda _path: "file-hash")
+    monkeypatch.setattr(
+        pipeline, "is_file_already_ingested", lambda *_args, **_kwargs: True
+    )
+    monkeypatch.setattr(
+        pipeline, "_build_supabase_client", lambda *_args, **_kwargs: fake_supabase
+    )
+    monkeypatch.setattr(
+        pipeline,
+        "_build_service_supabase_client",
+        lambda *_args, **_kwargs: fake_supabase,
+    )
+    monkeypatch.setattr(
+        pipeline, "partition_document", lambda _path: [FakeElement("x" * 100)]
+    )
+    monkeypatch.setattr(pipeline, "extract_images_from_pdf", lambda _path: {})
+
+    def fake_extract_document_entities(
+        elements, access_token=None, forced_category=None
+    ):
+        del elements, access_token
+        captured["forced_category"] = forced_category
+        return SimpleNamespace(is_allowed=True, document_type=forced_category)
+
+    monkeypatch.setattr(
+        pipeline, "extract_document_entities", fake_extract_document_entities
+    )
+    monkeypatch.setattr(
+        pipeline, "create_chunks", lambda elements, text_chars=None: ["chunk"]
+    )
+    monkeypatch.setattr(
+        pipeline,
+        "process_chunks",
+        lambda *args, **kwargs: (
+            [Document(page_content="body", metadata={"source": "Test Doc"})],
+            ["doc-1"],
+        ),
+    )
+    monkeypatch.setattr(
+        pipeline, "build_raptor_tree", lambda docs, ids, user_id: (docs, ids)
+    )
+    monkeypatch.setattr(pipeline, "upload_to_supabase", lambda *args, **kwargs: None)
+    monkeypatch.setattr(
+        pipeline, "invalidate_user_cache", lambda *args, **kwargs: None
+    )
+
+    result = pipeline.run_ingestion(
+        pdf_path="file.pdf",
+        force=True,
+        original_filename="file.pdf",
+        access_token="token",
+    )
+
+    assert result["document_type"] == "short_story"
+    assert captured["forced_category"] == "short_story"
+
+    select_idx = fake_supabase.ops.index(
+        ("ingested_files", "select", {"user_id": "user-1", "file_hash": "file-hash", "limit": 1})
+    )
+    delete_idx = fake_supabase.ops.index(
+        ("ingested_files", "delete", {"user_id": "user-1", "file_hash": "file-hash"})
+    )
+    assert select_idx < delete_idx
+
+
+def test_upload_to_supabase_uses_batch_rpc_and_skips_success_sleep(monkeypatch):
+    calls = []
+    sleeps = []
+
+    class FakeRpc:
+        def __init__(self, name, params):
+            self.name = name
+            self.params = params
+
+        def execute(self):
+            calls.append((self.name, self.params))
+            return SimpleNamespace(data=[])
+
+    class FakeBatchSupabase:
+        def rpc(self, name, params):
+            return FakeRpc(name, params)
+
+    class FakeEmbedder:
+        def embed_documents(self, texts):
+            return [[float(i), float(len(text))] for i, text in enumerate(texts, 1)]
+
+    monkeypatch.setattr(auth_utils, "safe_extract_jwt_sub", lambda _token: "user-1")
+    monkeypatch.setattr(pipeline, "_build_embeddings", lambda: FakeEmbedder())
+    monkeypatch.setattr(
+        pipeline,
+        "_build_service_supabase_client",
+        lambda *_args, **_kwargs: FakeBatchSupabase(),
+    )
+    monkeypatch.setattr(pipeline.time, "sleep", lambda seconds: sleeps.append(seconds))
+
+    docs = [
+        Document(page_content="alpha", metadata={"source": "A", "node_type": "leaf"}),
+        Document(page_content="beta", metadata={"source": "B", "node_type": "summary"}),
+    ]
+
+    pipeline.upload_to_supabase(
+        docs,
+        ["doc-1", "doc-2"],
+        access_token="token",
+    )
+
+    assert calls
+    assert calls[0][0] == "insert_document_chunks_batch"
+    assert len(calls[0][1]["p_rows"]) == 2
+    assert sleeps == []
+
+
+def test_run_ingestion_records_stage_timing_events(monkeypatch):
+    fake_supabase = FakeIngestionSupabase()
+
+    monkeypatch.setattr(auth_utils, "extract_jwt_sub", lambda _token: "user-1")
+    monkeypatch.setattr(pipeline, "get_file_fingerprint", lambda _path: "file-hash")
+    monkeypatch.setattr(
+        pipeline, "is_file_already_ingested", lambda *_args, **_kwargs: False
+    )
+    monkeypatch.setattr(
+        pipeline, "_build_supabase_client", lambda *_args, **_kwargs: fake_supabase
+    )
+    monkeypatch.setattr(
+        pipeline,
+        "_build_service_supabase_client",
+        lambda *_args, **_kwargs: fake_supabase,
+    )
+    monkeypatch.setattr(
+        pipeline, "partition_document", lambda _path: [FakeElement("x" * 120)]
+    )
+    monkeypatch.setattr(pipeline, "extract_images_from_pdf", lambda _path: {})
+    monkeypatch.setattr(
+        pipeline,
+        "extract_document_entities",
+        lambda *args, **kwargs: SimpleNamespace(
+            is_allowed=True,
+            document_type="short_story",
+            primary_topics=[],
+            brief_summary="Short story",
+            key_entities=[],
+        ),
+    )
+    monkeypatch.setattr(pipeline, "create_chunks", lambda elements, text_chars=None: ["chunk"])
+    monkeypatch.setattr(
+        pipeline,
+        "process_chunks",
+        lambda *args, **kwargs: (
+            [Document(page_content="body", metadata={"source": "Test Doc"})],
+            ["doc-1"],
+        ),
+    )
+    monkeypatch.setattr(
+        pipeline, "build_raptor_tree", lambda docs, ids, user_id: (docs, ids)
+    )
+    monkeypatch.setattr(pipeline, "upload_to_supabase", lambda *args, **kwargs: None)
+    monkeypatch.setattr(
+        pipeline, "invalidate_user_cache", lambda *args, **kwargs: None
+    )
+
+    pipeline.run_ingestion(
+        pdf_path="file.pdf",
+        force=False,
+        original_filename="file.pdf",
+        access_token="token",
+    )
+
+    stage_rows = [
+        payload
+        for name, payload in fake_supabase.inserts
+        if name == "ingestion_retry_logs" and payload.get("event_type") == "stage_timing"
+    ]
+    stages = {payload["message"] for payload in stage_rows}
+    assert any('"stage": "partition"' in stage for stage in stages)
+    assert any('"stage": "classify"' in stage for stage in stages)
+    assert any('"stage": "chunk_process"' in stage for stage in stages)
+    assert any('"stage": "raptor"' in stage for stage in stages)
+    assert any('"stage": "upload"' in stage for stage in stages)
+
+
+def test_recover_or_prepare_orphaned_upload_repairs_completed_upload(monkeypatch):
+    fake_service = FakeRecoverySupabase(
+        documents=[
+            {
+                "user_id": "user-1",
+                "content": "Abdul Manan — Deep Foundations Guide",
+                "metadata": {
+                    "file_hash": "file-hash",
+                    "source": "Recovered Guide",
+                    "document_type": "technical_guide",
+                },
+            },
+            {
+                "user_id": "user-1",
+                "content": 'The "Why Before What" Bible for ML/DL/AI Engineering',
+                "metadata": {
+                    "file_hash": "file-hash",
+                    "source": "Recovered Guide",
+                    "document_type": "technical_guide",
+                },
+            },
+        ],
+        ingestion_logs=[
+            {
+                "user_id": "user-1",
+                "file_hash": "file-hash",
+                "event_type": "upload_complete",
+            }
+        ],
+    )
+    monkeypatch.setattr(
+        pipeline, "_build_service_supabase_client", lambda *_args, **_kwargs: fake_service
+    )
+
+    result = pipeline._recover_or_prepare_orphaned_upload(
+        "file-hash",
+        user_id="user-1",
+        filename_hint="fallback.pdf",
+    )
+
+    assert result["recovered_existing"] is True
+    upsert = next(item for item in fake_service.upserts if item[0] == "ingested_files")
+    assert upsert[1]["file_hash"] == "file-hash"
+    assert upsert[1]["document_type"] == "technical_guide"
+    assert upsert[1]["chunk_count"] == 2
+
+
+def test_run_ingestion_short_circuits_on_recovered_existing_upload(monkeypatch):
+    monkeypatch.setattr(auth_utils, "extract_jwt_sub", lambda _token: "user-1")
+    monkeypatch.setattr(pipeline, "get_file_fingerprint", lambda _path: "file-hash")
+    monkeypatch.setattr(
+        pipeline, "is_file_already_ingested", lambda *_args, **_kwargs: False
+    )
+    monkeypatch.setattr(
+        pipeline,
+        "_recover_or_prepare_orphaned_upload",
+        lambda *_args, **_kwargs: {
+            "pending_review": False,
+            "document_type": "technical_guide",
+            "filename": "Recovered Guide",
+            "file_hash": "file-hash",
+            "recovered_existing": True,
+        },
+    )
+    monkeypatch.setattr(
+        pipeline,
+        "partition_document",
+        lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("should not recompute")),
+    )
+
+    result = pipeline.run_ingestion(
+        pdf_path="file.pdf",
+        force=False,
+        original_filename="file.pdf",
+        access_token="token",
+    )
+
+    assert result["recovered_existing"] is True
+    assert result["file_hash"] == "file-hash"
+
+
+def test_identity_json_extracts_cover_metadata():
+    identity = pipeline._identity_json_from_elements(
+        [
+            FakeElement("Abdul Manan — Deep Foundations Guide", page_number=1),
+            FakeElement('The "Why Before What" Bible for ML/DL/AI Engineering', page_number=1),
+            FakeElement(
+                "This guide exists because knowing definitions is not enough. Most people learn ML backwards.",
+                page_number=1,
+            ),
+        ],
+        fallback_title="Fallback Guide",
+    )
+
+    # Display title should be the actual title line; the personalized cover-owner line is stored separately.
+    assert identity["display_title"] == 'The "Why Before What" Bible for ML/DL/AI Engineering'
+    assert identity["subtitle"] == "This guide exists because knowing definitions is not enough. Most people learn ML backwards."
+    assert identity["named_owner"] == "Abdul Manan"
+    assert "knowing definitions is not enough" in identity["opening_page_summary"].lower()
+    assert identity["field_presence"]["publisher"] is False
+
+
+def test_identity_json_strips_null_bytes_from_opening_page_fields():
+    identity = pipeline._identity_json_from_elements(
+        [
+            FakeElement("Abdul\x00 Manan — Deep Foundations Guide", page_number=1),
+            FakeElement('The "Why Before What"\x00 Bible for ML/DL/AI Engineering', page_number=1),
+            FakeElement("Publisher:\x00 Not stated", page_number=1),
+        ],
+        fallback_title="Fallback Guide",
+    )
+
+    serialized = json.dumps(identity)
+    assert "\u0000" not in serialized
+    assert "\x00" not in identity["display_title"]
+    assert "\x00" not in identity["subtitle"]
+    assert "\x00" not in identity["cover_text"]
+
+
+def test_identity_json_from_docs_dedupes_repeated_opening_page_content():
+    repeated_row = {
+        "content": (
+            "Abdul Manan — Deep Foundations Guide\n"
+            'The "Why Before What" Bible for ML/DL/AI Engineering\n'
+            "This guide exists because knowing definitions is not enough."
+        ),
+        "metadata": {
+            "page_numbers": [1],
+            "original_content": {
+                "raw_text": (
+                    "Abdul Manan — Deep Foundations Guide\n"
+                    'The "Why Before What" Bible for ML/DL/AI Engineering\n'
+                    "This guide exists because knowing definitions is not enough."
+                )
+            },
+        },
+    }
+
+    identity = pipeline._identity_json_from_docs(
+        [repeated_row, repeated_row],
+        fallback_title="Fallback Guide",
+    )
+
+    assert identity["cover_text"].count("Abdul Manan — Deep Foundations Guide") == 1
+    assert (
+        identity["opening_page_summary"].count(
+            'The "Why Before What" Bible for ML/DL/AI Engineering'
+        )
+        == 1
+    )
+
+
+def test_classify_query_route_decision_marks_exact_fact_query():
+    decision = pipeline._classify_query_route_decision(
+        "Whose guide is this? Answer using the exact name written in the document."
+    )
+
+    assert decision.route_class == "exact_fact"
+    assert decision.exact_field == "owner"
+    assert decision.preserve_query is True
+    assert decision.disable_memory is True
+
+
+def test_classify_query_route_decision_marks_page_scoped_query():
+    decision = pipeline._classify_query_route_decision(
+        "Summarize only the first page, not the whole guide."
+    )
+
+    assert decision.route_class == "page_scoped"
+    assert decision.page_scope == "first_page"
+    assert decision.preserve_query is True
+    assert decision.disable_memory is True
+
+
+class FakeAmbiguityTable:
+    def __init__(self, rows):
+        self.rows = rows
+        self.filters = {}
+        self.action = None
+
+    def select(self, *_args):
+        self.action = "select"
+        return self
+
+    def eq(self, key, value):
+        self.filters[key] = value
+        return self
+
+    def execute(self):
+        return SimpleNamespace(data=list(self.rows))
+
+
+class FakeAmbiguityRpc:
+    def __init__(self, supabase, params):
+        self.supabase = supabase
+        self.params = params
+
+    def execute(self):
+        self.supabase.rpc_calls.append(self.params)
+        fhash = (self.params.get("filter") or {}).get("file_hash")
+        score = 0.22 if fhash == "A" else 0.11
+        return SimpleNamespace(data=[{"combined_score": score}])
+
+
+class FakeAmbiguitySupabase:
+    def __init__(self, rows):
+        self.rows = rows
+        self.rpc_calls = []
+
+    def table(self, _name: str):
+        return FakeAmbiguityTable(self.rows)
+
+    def rpc(self, _name: str, params):
+        return FakeAmbiguityRpc(self, params)
+
+
+def test_check_query_ambiguity_forces_clarification_for_identity_queries_in_multi_doc_scope(monkeypatch):
+    fake = FakeAmbiguitySupabase(
+        rows=[
+            {"file_hash": "A", "filename": "Guide A.pdf"},
+            {"file_hash": "B", "filename": "Guide B.pdf"},
+        ]
+    )
+    monkeypatch.setattr(pipeline, "_build_supabase_client", lambda *_args, **_kwargs: fake)
+
+    res = pipeline.check_query_ambiguity("Whose guide is this?", access_token=None, category="All")
+
+    assert res["is_ambiguous"] is True
+    assert res["top_file_hash"] is None
+    assert res["clarification_options"]
+    assert fake.rpc_calls == []
+
+
+def test_check_query_ambiguity_rpc_includes_p_user_id_to_avoid_overload(monkeypatch):
+    fake = FakeAmbiguitySupabase(
+        rows=[
+            {"file_hash": "A", "filename": "Doc A.pdf"},
+            {"file_hash": "B", "filename": "Doc B.pdf"},
+        ]
+    )
+    monkeypatch.setattr(pipeline, "_build_supabase_client", lambda *_args, **_kwargs: fake)
+
+    res = pipeline.check_query_ambiguity("summarize the document", access_token=None, category="All")
+
+    assert res["is_ambiguous"] is True
+    assert fake.rpc_calls, "Expected ambiguity scoring RPC calls"
+    assert all("p_user_id" in call for call in fake.rpc_calls)
+
+
+def test_check_query_ambiguity_autopins_single_doc_in_category_even_for_identity_query(monkeypatch):
+    fake = FakeAmbiguitySupabase(
+        rows=[
+            {"file_hash": "ONLY", "filename": "Only Doc.pdf"},
+        ]
+    )
+    monkeypatch.setattr(pipeline, "_build_supabase_client", lambda *_args, **_kwargs: fake)
+
+    res = pipeline.check_query_ambiguity(
+        "Whose guide is this?",
+        access_token=None,
+        category="technical_guide",
+    )
+
+    assert res["is_ambiguous"] is False
+    assert res["top_file_hash"] == "ONLY"
+
+
+def test_check_query_ambiguity_lists_only_three_options_when_many_docs(monkeypatch):
+    fake = FakeAmbiguitySupabase(
+        rows=[
+            {"file_hash": "A", "filename": "A.pdf"},
+            {"file_hash": "B", "filename": "B.pdf"},
+            {"file_hash": "C", "filename": "C.pdf"},
+            {"file_hash": "D", "filename": "D.pdf"},
+            {"file_hash": "E", "filename": "E.pdf"},
+        ]
+    )
+    monkeypatch.setattr(pipeline, "_build_supabase_client", lambda *_args, **_kwargs: fake)
+
+    res = pipeline.check_query_ambiguity("What is the exact full title?", access_token=None, category="All")
+
+    assert res["is_ambiguous"] is True
+    assert len(res.get("clarification_options") or []) == 3
+
+
+def test_query_followup_guard_detects_ordinal_without_enumeration():
+    assert query_api._contains_ordinal_followup("What about the second one?") is True
+    assert query_api._history_has_explicit_enumeration(
+        [{"role": "assistant", "content": "No list here."}]
+    ) is False
+
+
+def test_query_followup_guard_allows_ordinal_when_prior_answer_lists_items():
+    history = [
+        {"role": "assistant", "content": "1. Alice\n2. Bob\n"},
+    ]
+    assert query_api._contains_ordinal_followup("What about the second one?") is True
+    assert query_api._history_has_explicit_enumeration(history) is True
+
+
+def test_generate_sub_queries_skips_rewrite_for_exact_fact(monkeypatch):
+    monkeypatch.setattr(
+        providers.ProviderFactory,
+        "build_chat_llm",
+        staticmethod(lambda **_kwargs: (_ for _ in ()).throw(AssertionError("rewriter should not be called"))),
+    )
+
+    queries = pipeline.generate_sub_queries(
+        "What is the exact full title of this guide?",
+        route_class="exact_fact",
+    )
+
+    assert queries == ["What is the exact full title of this guide?"]
+
+
+def test_identity_documents_for_query_answers_not_stated_publisher():
+    row = {
+        "filename": "Guide.pdf",
+        "identity_json": {
+            "display_title": "Abdul Manan — Deep Foundations Guide",
+            "field_presence": {"publisher": False},
+            "source_pages": [1],
+        },
+    }
+    route_decision = pipeline.RouteDecision(
+        route_class="exact_fact",
+        route_reason="identity_field:publisher",
+        exact_field="publisher",
+    )
+
+    docs = pipeline._identity_documents_for_query(
+        row,
+        query="Does this guide explicitly name a publisher on the opening pages? If not, say not stated.",
+        route_decision=route_decision,
+    )
+
+    assert len(docs) == 1
+    assert "not stated on the opening pages" in docs[0].page_content.lower()
+    assert docs[0].metadata["retrieval_branch"] == "identity_store"
+
+
+def test_build_history_block_returns_structured_state_without_role_labels():
+    block = pipeline._build_history_block(
+        [
+            {"role": "user", "content": "Whose guide is this?"},
+            {"role": "assistant", "content": "ASSISTANT: Abdul Manan — Deep Foundations Guide."},
+        ],
+        route_class="factoid",
+        eval_mode=False,
+    )
+
+    assert "CONVERSATION STATE:" in block
+    assert "previous_user_intent:" in block
+    assert "previous_answer_summary:" in block
+    assert "ASSISTANT:" not in block
+    assert "USER:" not in block
+
+
+def test_save_to_memory_writes_structured_payloads(monkeypatch):
+    fake_service = FakeServiceSupabase()
+    monkeypatch.setattr(
+        pipeline, "_build_service_supabase_client", lambda *_args, **_kwargs: fake_service
+    )
+    monkeypatch.setattr(pipeline, "_stable_user_id", lambda *_args, **_kwargs: "user-1")
+    monkeypatch.setattr(pipeline, "get_cached_embedding", lambda _text: [0.1, 0.2])
+
+    chunks = [
+        Document(
+            page_content="body",
+            metadata={"file_hash": "file-1", "document_type": "machine_learning_guide"},
+        )
+    ]
+
+    pipeline._save_to_memory(
+        "session-1",
+        "Whose guide is this?",
+        "Abdul Manan — Deep Foundations Guide\n\n---\n**Sources:**\n[Source 1]",
+        access_token=None,
+        route_class="factoid",
+        chunks=chunks,
+    )
+
+    assert len(fake_service.inserts) == 2
+    user_payload = json.loads(fake_service.inserts[0][1]["content"])
+    answer_payload = json.loads(fake_service.inserts[1][1]["content"])
+    assert user_payload["kind"] == "user_query"
+    assert answer_payload["kind"] == "assistant_fact"
+    assert answer_payload["file_hashes"] == ["file-1"]
+    assert "Sources" not in answer_payload["summary"]
+
+
+def test_generate_answer_stream_eval_mode_skips_history_and_memory_injection(monkeypatch):
+    captured = {}
+
+    class FakeLLM:
+        async def astream(self, messages):
+            captured["prompt"] = messages[0].content[0]["text"]
+            yield SimpleNamespace(content="clean answer")
+
+    monkeypatch.setattr(pipeline, "_build_llm", lambda needs_vision=False: FakeLLM())
+    monkeypatch.setattr(
+        pipeline,
+        "_get_episodic_memory",
+        lambda *args, **kwargs: "" if kwargs.get("eval_mode") else "SESSION FACTS:\n- prior answer: x\n",
+    )
+    monkeypatch.setattr(pipeline, "_log_retrieval_reward", lambda *args, **kwargs: None)
+    monkeypatch.setattr(pipeline, "_save_to_memory", lambda *args, **kwargs: None)
+    monkeypatch.setattr(pipeline, "_persist_query_trace", lambda **_kwargs: "trace-1")
+
+    chunk = Document(
+        page_content="Body text",
+        metadata={
+            "source": "Guide",
+            "chunk_index": 1,
+            "document_type": "machine_learning_guide",
+            "relevance_score": 0.9,
+            "route_class": "factoid",
+            "original_content": {"raw_text": "Body text", "tables_html": []},
+        },
+    )
+
+    async def collect():
+        events = []
+        async for event in pipeline.generate_answer_stream(
+            chunks=[chunk],
+            query="Tell me more",
+            chat_history=[
+                {"role": "user", "content": "Who is this guide for?"},
+                {"role": "assistant", "content": "It is personalized."},
+            ],
+            session_id="session-1",
+            eval_mode=True,
+        ):
+            events.append(event)
+        return events
+
+    events = asyncio.run(collect())
+
+    assert any(event["type"] == "done" for event in events)
+    assert "CONVERSATION STATE:" not in captured["prompt"]
+    assert "SESSION FACTS:" not in captured["prompt"]
+
+
+def test_persist_query_trace_marks_output_echo_and_contamination(monkeypatch):
+    fake_service = FakeServiceSupabase()
+    monkeypatch.setattr(
+        pipeline, "_build_service_supabase_client", lambda *_args, **_kwargs: fake_service
+    )
+    monkeypatch.setattr(pipeline, "_persist_trace_graph_enrichment", lambda *args, **kwargs: None)
+
+    chunks = [
+        Document(
+            page_content="body",
+            metadata={
+                "trace_id": "trace-echo",
+                "route_class": "factoid",
+                "route_mode": "default",
+                "source": "Guide",
+                "document_type": "machine_learning_guide",
+                "trace_quality": {
+                    "retrieval_relevance_proxy": 0.8,
+                    "history_injected": True,
+                    "memory_injected": True,
+                },
+            },
+        )
+    ]
+
+    pipeline._persist_query_trace(
+        query="Why does this guide say it exists?",
+        session_id="session-1",
+        chunks=chunks,
+        answer="ASSISTANT: This guide exists because knowing definitions is not enough.",
+        access_token=None,
+    )
+
+    upsert = next(item for item in fake_service.upserts if item[0] == "query_traces")
+    failure_modes = set(upsert[1]["failure_modes"])
+    assert {"output_echo", "history_contamination", "memory_contamination"} <= failure_modes
+
+
+def test_retrieve_chunks_exact_fact_prefers_identity_store(monkeypatch):
+    monkeypatch.setattr(pipeline, "_stable_user_id", lambda *_args, **_kwargs: "user-1")
+    monkeypatch.setattr(pipeline, "_route_query_experts", lambda *args, **kwargs: {
+        "selected_experts": ["dense_chunk"],
+        "expert_weights": {"dense_chunk": 1.0},
+        "confidence": 0.9,
+    })
+    monkeypatch.setattr(
+        pipeline,
+        "_load_or_backfill_identity_row",
+        lambda *args, **kwargs: {
+            "filename": "Guide.pdf",
+            "identity_json": {
+                "display_title": "Abdul Manan — Deep Foundations Guide",
+                "subtitle": 'The "Why Before What" Bible for ML/DL/AI Engineering',
+                "named_owner": "Abdul Manan",
+                "field_presence": {"owner": True},
+                "source_pages": [1],
+            },
+        },
+    )
+
+    docs = pipeline.retrieve_chunks(
+        query="Whose guide is this? Answer using the exact name written in the document.",
+        original_query="Whose guide is this? Answer using the exact name written in the document.",
+        user_id="user-1",
+        priority_file_hashes=["file-1"],
+    )
+
+    assert len(docs) == 1
+    assert docs[0].metadata["retrieval_branch"] == "identity_store"
+    assert "Abdul Manan" in docs[0].page_content
+
+
+def test_multi_doc_context_budget_preserves_one_chunk_per_pinned_doc(monkeypatch):
+    monkeypatch.setattr(
+        pipeline, "_build_service_supabase_client", lambda *_args, **_kwargs: FakeRetrieveSupabase()
+    )
+    monkeypatch.setattr(pipeline, "get_cached_embedding", lambda _query: [0.1, 0.2])
+    monkeypatch.setattr(pipeline.cohere, "Client", FakeCohereClient)
+    monkeypatch.setattr(pipeline, "_log_rerank_feedback", lambda *args, **kwargs: None)
+    monkeypatch.setattr(pipeline.config, "MAX_CONTEXT_CHARS", 700, raising=False)
+
+    docs = pipeline.retrieve_chunks(
+        query="compare the themes of both stories",
+        category="short_story",
+        access_token="token",
+        priority_file_hashes=["A", "B"],
+    )
+
+    assert len(docs) == 2
+    assert {doc.metadata["file_hash"] for doc in docs} == {"A", "B"}
+
+
+def test_build_pinned_query_plan_scopes_title_queries_to_own_doc():
+    plan = pipeline._build_pinned_query_plan(
+        "summarise the story short story",
+        [
+            {"file_hash": "A", "filename": "About Love Anton Chekhov"},
+            {"file_hash": "B", "filename": "BEYOND BOUNDS"},
+        ],
+        "generic_pinned",
+    )
+
+    about_entries = [entry for entry in plan if "About Love Anton Chekhov" in entry["query_text"]]
+    beyond_entries = [entry for entry in plan if "BEYOND BOUNDS" in entry["query_text"]]
+
+    assert about_entries
+    assert beyond_entries
+    assert all(entry["target_file_hashes"] == ["A"] for entry in about_entries)
+    assert all(entry["target_file_hashes"] == ["B"] for entry in beyond_entries)
+
+
+def test_partition_document_retries_with_hi_res_when_fast_is_suspiciously_thin(monkeypatch):
+    calls = []
+
+    def fake_partition_pdf(*, filename, strategy, **_kwargs):
+        del filename
+        calls.append(strategy)
+        if strategy == "fast":
+            return [FakeElement("x" * 50, page_number=1)]
+        return [
+            FakeElement("x" * 500, page_number=1),
+            FakeElement("y" * 500, page_number=1),
+            FakeElement("z" * 500, page_number=1),
+        ]
+
+    monkeypatch.setattr(pipeline, "_has_text_layer", lambda _path: True)
+    monkeypatch.setattr(pipeline, "partition_pdf", fake_partition_pdf)
+
+    elements = pipeline.partition_document("file.pdf")
+
+    assert calls == ["fast", "hi_res"]
+    assert len(elements) == 3
+
+
+def test_create_chunks_splits_single_thin_narrative(monkeypatch):
+    long_text = (
+        '"Every single night..." Lee said softly. '
+        "The same demons kept returning, and the weight of them was unbearable. "
+        "She kept remembering the dream, the corridor, the whispering, and the crushing fear. "
+        "Classes were slipping away from her, and every conversation with the doctor felt more urgent. "
+        "Still, she tried to describe what she saw, heard, and felt in careful detail."
+    ) * 3
+
+    def fake_chunk_by_title(elements, **_kwargs):
+        del elements
+        return [
+            SimpleNamespace(
+                text=long_text,
+                metadata=SimpleNamespace(orig_elements=[FakeElement(long_text)]),
+            )
+        ]
+
+    monkeypatch.setattr(pipeline, "chunk_by_title", fake_chunk_by_title)
+
+    chunks = pipeline.create_chunks([FakeElement(long_text)], text_chars=len(long_text))
+
+    assert len(chunks) >= 2
+    assert all(getattr(chunk, "text", "") for chunk in chunks)
+
+
+def test_build_raptor_tree_synthesizes_root_for_single_leaf(monkeypatch):
+    class FakeLLM:
+        def invoke(self, _messages):
+            return SimpleNamespace(content="Root summary")
+
+    monkeypatch.setattr(pipeline, "_build_llm", lambda **_kwargs: FakeLLM())
+    leaf = Document(
+        page_content="Leaf content",
+        metadata={
+            "source": "BEYOND BOUNDS",
+            "file_hash": "B",
+            "document_type": "short_story",
+            "summary": "Leaf summary",
+            "chunk_index": 1,
+            "page_numbers": [1],
+        },
+    )
+
+    docs, ids = pipeline.build_raptor_tree([leaf], ["leaf-1"], "user-1")
+
+    assert len(docs) == 2
+    assert len(ids) == 2
+    assert any(doc.metadata.get("node_type") == "summary" for doc in docs)
+
+
+def test_generic_multi_doc_mode_keeps_weak_doc_with_candidates(monkeypatch):
+    monkeypatch.setattr(
+        pipeline, "_build_service_supabase_client", lambda *_args, **_kwargs: FakeRetrieveSupabase()
+    )
+    monkeypatch.setattr(pipeline, "get_cached_embedding", lambda _query: [0.1, 0.2])
+    monkeypatch.setattr(pipeline.cohere, "Client", FakeCohereClient)
+    monkeypatch.setattr(pipeline, "_log_rerank_feedback", lambda *args, **kwargs: None)
+    monkeypatch.setattr(pipeline.config, "MAX_CONTEXT_CHARS", 2_000, raising=False)
+
+    docs = pipeline.retrieve_chunks(
+        query="summarise the story short story",
+        category="short_story",
+        access_token="token",
+        priority_file_hashes=["A", "B"],
+        original_query="summarise the story",
+    )
+
+    assert {doc.metadata["file_hash"] for doc in docs} == {"A", "B"}
+    assert docs[0].metadata["route_mode"] == "generic_pinned"
+    assert len(docs[0].metadata["doc_diagnostics"]) == 2
+
+
+def test_weighted_doc_prior_fusion_does_not_saturate_scores():
+    fused = pipeline._combine_local_and_doc_score(0.95, 1.0, 0.2)
+    assert fused < 1.0
+    assert fused == 0.96
+
+
+def test_generate_answer_stream_done_event_includes_trace_metadata(monkeypatch):
+    class FakeLLM:
+        async def astream(self, _messages):
+            yield SimpleNamespace(content="ok")
+
+    monkeypatch.setattr(pipeline, "_build_llm", lambda needs_vision=False: FakeLLM())
+    monkeypatch.setattr(pipeline, "_get_episodic_memory", lambda *args, **kwargs: "")
+    monkeypatch.setattr(pipeline, "_log_retrieval_reward", lambda *args, **kwargs: None)
+    monkeypatch.setattr(pipeline, "_save_to_memory", lambda *args, **kwargs: None)
+    monkeypatch.setattr(pipeline, "_persist_query_trace", lambda **_kwargs: "trace-123")
+
+    chunk = Document(
+        page_content="Leaf fallback",
+        metadata={
+            "source": "About Love Anton Chekhov",
+            "chunk_index": 1,
+            "document_type": "short_story",
+            "relevance_score": 0.6,
+            "original_content": {"raw_text": "Leaf raw text", "tables_html": []},
+            "trace_id": "trace-123",
+            "route_mode": "explicit_compare",
+            "doc_diagnostics": [{"file_hash": "A", "source": "About Love Anton Chekhov", "included": True, "candidate_count": 2, "doc_score": 0.6, "confidence_label": "high", "reason": "supported"}],
+        },
+    )
+
+    async def collect():
+        events = []
+        async for event in pipeline.generate_answer_stream(
+            chunks=[chunk],
+            query="compare the themes",
+            access_token=None,
+            category="short_story",
+            priority_file_hashes=["A", "B"],
+        ):
+            events.append(event)
+        return events
+
+    events = asyncio.run(collect())
+    done_event = next(event for event in events if event["type"] == "done")
+
+    assert done_event["trace_id"] == "trace-123"
+    assert done_event["doc_diagnostics"][0]["source"] == "About Love Anton Chekhov"
+
+
+def test_generate_answer_stream_sanitizes_template_tokens_and_records_metrics(monkeypatch):
+    captured = {}
+
+    def fake_persist_query_trace(**kwargs):
+        captured["kwargs"] = kwargs
+        return "trace-xyz"
+
+    class FakeLLM:
+        async def astream(self, _messages):
+            yield SimpleNamespace(content="assistant<|header_end|>Hello")
+            yield SimpleNamespace(content=" there<|eot_id|>")
+
+    monkeypatch.setattr(pipeline, "_build_llm", lambda needs_vision=False: FakeLLM())
+    monkeypatch.setattr(pipeline, "_get_episodic_memory", lambda *args, **kwargs: "")
+    monkeypatch.setattr(pipeline, "_log_retrieval_reward", lambda *args, **kwargs: None)
+    monkeypatch.setattr(pipeline, "_save_to_memory", lambda *args, **kwargs: None)
+    monkeypatch.setattr(pipeline, "_persist_query_trace", fake_persist_query_trace)
+
+    chunk = Document(
+        page_content="Leaf fallback",
+        metadata={
+            "source": "About Love Anton Chekhov",
+            "chunk_index": 1,
+            "document_type": "short_story",
+            "relevance_score": 0.6,
+            "original_content": {"raw_text": "Leaf raw text", "tables_html": []},
+            "trace_id": "trace-xyz",
+            "route_mode": "default",
+        },
+    )
+
+    async def collect():
+        events = []
+        async for event in pipeline.generate_answer_stream(
+            chunks=[chunk],
+            query="hello",
+            access_token=None,
+            category="short_story",
+        ):
+            events.append(event)
+        return events
+
+    events = asyncio.run(collect())
+    tokens = "".join(event["content"] for event in events if event["type"] == "token")
+
+    assert "<|" not in tokens
+    assert "Hello there" in tokens
+    assert captured["kwargs"]["sanitizer_metrics"]["sanitizer_triggered"] is True
+    assert captured["kwargs"]["sanitizer_metrics"]["sanitized_token_count"] > 0
+
+
+def test_duplicate_chunk_collapse_removes_overlap():
+    kept, collapsed = pipeline._collapse_near_duplicate_candidates(
+        [
+            {
+                "id": "a",
+                "content": "Alpha beta gamma delta epsilon zeta",
+                "metadata": {"file_hash": "doc-a", "source": "Doc A"},
+            },
+            {
+                "id": "b",
+                "content": "Alpha beta gamma delta epsilon zeta eta",
+                "metadata": {"file_hash": "doc-a", "source": "Doc A"},
+            },
+            {
+                "id": "c",
+                "content": "Completely different content",
+                "metadata": {"file_hash": "doc-a", "source": "Doc A"},
+            },
+        ]
+    )
+
+    assert collapsed == 1
+    assert [row["id"] for row in kept] == ["a", "c"]
+
+
+def test_analyse_intent_rewrites_follow_up_query(monkeypatch):
+    monkeypatch.setattr(
+        pipeline.intent_classifier,
+        "predict",
+        lambda *_args, **_kwargs: {
+            "needs_clarification": False,
+            "confidence": 0.95,
+        },
+    )
+    monkeypatch.setattr(
+        pipeline.intent_classifier,
+        "record_feedback",
+        lambda *args, **kwargs: None,
+    )
+
+    session_key = pipeline._session_cache_key("sess-1", user_id="user-1")
+    pipeline._last_query_context[session_key] = {
+        "query": "Compare About Love and BEYOND BOUNDS",
+        "updated_at": pipeline.time.time(),
+    }
+    pipeline._last_chunks[session_key] = [Document(page_content="cached", metadata={})]
+
+    result = pipeline.analyse_intent(
+        query="What about the second one?",
+        category="All",
+        chat_history=[
+            {"role": "user", "content": "Compare About Love and BEYOND BOUNDS"},
+            {"role": "assistant", "content": "Here is the comparison."},
+        ],
+        session_id="sess-1",
+        user_id="user-1",
+    )
+
+    assert result["route_class"] == "follow_up"
+    assert "follow-up about: Compare About Love and BEYOND BOUNDS" in result["enriched_query"]
+
+    pipeline._last_query_context.pop(session_key, None)
+    pipeline._last_chunks.pop(session_key, None)
+
+
+def test_record_answer_feedback_persists_feedback_and_promotes(monkeypatch):
+    fake_service = FakeServiceSupabase()
+    fake_service.trace_rows.append(
+        {"trace_id": "8f8c1f3f-bcb6-43a8-b10d-85f31a917111", "session_id": "sess-1", "question": "What is common?"}
+    )
+    monkeypatch.setattr(
+        pipeline, "_build_service_supabase_client", lambda *_args, **_kwargs: fake_service
+    )
+
+    ok = pipeline.record_answer_feedback(
+        {
+            "trace_id": "8f8c1f3f-bcb6-43a8-b10d-85f31a917111",
+            "helpful": False,
+            "reason_code": "needs_improvement",
+            "correction_text": "The two stories should not be merged.",
+        },
+        access_token=None,
+    )
+
+    assert ok is True
+    feedback_insert = next(item for item in fake_service.inserts if item[0] == "answer_feedback")
+    assert feedback_insert[1]["promote_to_eval"] is True
+    assert any(item[0] == "graph_nodes" for item in fake_service.upserts)
+    assert any(item[0] == "graph_edges" for item in fake_service.upserts)
+
+
+def test_load_feedback_dataset_candidates_promotes_feedback_traces(monkeypatch):
+    fake_service = FakeServiceSupabase()
+    fake_service.feedback_rows.append(
+        {
+            "trace_id": "8f8c1f3f-bcb6-43a8-b10d-85f31a917111",
+            "helpful": False,
+            "accepted": False,
+            "reason_code": "unsupported_commonality",
+            "correction_text": "Insufficient evidence for commonality.",
+            "promote_to_eval": True,
+            "user_id": "user-1",
+        }
+    )
+    fake_service.trace_rows.append(
+        {
+            "trace_id": "8f8c1f3f-bcb6-43a8-b10d-85f31a917111",
+            "question": "What is common between these two documents?",
+            "doc_diagnostics": [{"source": "BEYOND BOUNDS", "reason": "low_scoped_confidence"}],
+            "failure_modes": ["unsupported_commonality"],
+            "answer_preview": "The documents both explore emotion.",
+        }
+    )
+    monkeypatch.setattr(
+        pipeline, "_build_service_supabase_client", lambda *_args, **_kwargs: fake_service
+    )
+
+    rows = run_eval.load_feedback_dataset_candidates(None, "user-1", limit=10)
+
+    assert len(rows) == 1
+    assert rows[0]["trace_id"] == "8f8c1f3f-bcb6-43a8-b10d-85f31a917111"
+    assert rows[0]["gold_evidence_text"] == "Insufficient evidence for commonality."
+
+
+def test_router_weights_trigger_summary_branch_filters(monkeypatch):
+    class TrackingRpc:
+        def __init__(self, supabase):
+            self.supabase = supabase
+
+        def execute(self):
+            self.supabase.rpc_filters.append(self.supabase.params["filter"])
+            node_type = self.supabase.params["filter"].get("node_type")
+            if node_type == "summary":
+                return SimpleNamespace(
+                    data=[
+                        {
+                            "id": "sum-1",
+                            "content": "Synthetic summary content",
+                            "metadata": {
+                                "file_hash": "A",
+                                "source": "About Love Anton Chekhov",
+                                "chunk_index": "1-4",
+                                "document_type": "short_story",
+                                "node_type": "summary",
+                                "node_level": 1,
+                            },
+                        }
+                    ]
+                )
+            return SimpleNamespace(
+                data=[
+                    {
+                        "id": "leaf-1",
+                        "content": "Leaf content",
+                        "metadata": {
+                            "file_hash": "A",
+                            "source": "About Love Anton Chekhov",
+                            "chunk_index": 1,
+                            "document_type": "short_story",
+                            "node_type": "leaf",
+                        },
+                    }
+                ]
+            )
+
+    class TrackingSupabase:
+        def __init__(self):
+            self.rpc_filters = []
+            self.params = {}
+
+        def table(self, _name: str):
+            return FakeRetrieveTable(self, "ingested_files")
+
+        def rpc(self, _name: str, params):
+            self.params = params
+            return TrackingRpc(self)
+
+    tracking = TrackingSupabase()
+    monkeypatch.setattr(
+        pipeline, "_build_service_supabase_client", lambda *_args, **_kwargs: tracking
+    )
+    monkeypatch.setattr(pipeline, "get_cached_embedding", lambda _query: [0.1, 0.2])
+    monkeypatch.setattr(pipeline, "_route_query_experts", lambda *args, **kwargs: {
+        "expert_weights": {
+            "dense_chunk": 0.3,
+            "raptor_summary": 0.4,
+            "graph_traversal": 0.1,
+            "episodic_memory": 0.1,
+            "hybrid_compare": 0.1,
+        },
+        "selected_experts": ["dense_chunk", "raptor_summary"],
+        "confidence": 0.4,
+    })
+    monkeypatch.setattr(pipeline.cohere, "Client", FakeCohereClient)
+    monkeypatch.setattr(pipeline, "_log_rerank_feedback", lambda *args, **kwargs: None)
+
+    docs = pipeline.retrieve_chunks(
+        query="tell me more",
+        access_token="token",
+        original_query="tell me more",
+    )
+
+    assert docs
+    assert any(f.get("node_type") == "summary" for f in tracking.rpc_filters)
+
+
+def test_thin_doc_overview_prefers_synthetic_root_summary():
+    leaf = Document(
+        page_content="Leaf content",
+        metadata={
+            "file_hash": "B",
+            "source": "BEYOND BOUNDS",
+            "node_type": "leaf",
+            "relevance_score": 0.9,
+        },
+    )
+    root = Document(
+        page_content="Synthetic root summary",
+        metadata={
+            "file_hash": "B",
+            "source": "BEYOND BOUNDS",
+            "node_type": "summary",
+            "synthetic_root_summary": True,
+            "relevance_score": 0.4,
+        },
+    )
+
+    ordered, buckets, policy = pipeline._materialize_evidence_buckets(
+        [leaf, root],
+        query="summarise the story",
+        route_mode="single",
+        doc_title_map={"B": "BEYOND BOUNDS"},
+    )
+
+    assert ordered[0].metadata["synthetic_root_summary"] is True
+    assert buckets[0]["thin_doc"] is True
+    assert policy["summary_like"] is True
+
+
+def test_graph_candidates_return_two_hop_related_chunks(monkeypatch):
+    fake_graph = FakeGraphServiceSupabase()
+    fake_graph.graph_nodes = [
+        {"user_id": "user-1", "node_key": "entity:alehin", "node_type": "entity", "label": "Alehin", "payload": {"file_hash": "A"}},
+        {"user_id": "user-1", "node_key": "summary:root-a", "node_type": "summary", "label": "About Love Anton Chekhov :: 1-4", "payload": {"file_hash": "A", "chunk_index": "1-4"}},
+        {"user_id": "user-1", "node_key": "document:a", "node_type": "document", "label": "About Love Anton Chekhov", "payload": {"file_hash": "A"}},
+    ]
+    fake_graph.graph_edges = [
+        {"user_id": "user-1", "source_node_key": "entity:alehin", "target_node_key": "summary:root-a", "edge_type": "mentions", "weight": 1.0, "payload": {}},
+        {"user_id": "user-1", "source_node_key": "summary:root-a", "target_node_key": "document:a", "edge_type": "part_of", "weight": 1.0, "payload": {}},
+    ]
+    vector_rows = [
+        {
+            "id": "sum-1",
+            "user_id": "user-1",
+            "content": "Alehin appears in About Love.",
+            "metadata": {
+                "file_hash": "A",
+                "source": "About Love Anton Chekhov",
+                "node_type": "summary",
+                "chunk_index": "1-4",
+            },
+        }
+    ]
+
+    monkeypatch.setattr(
+        pipeline, "_build_service_supabase_client", lambda *_args, **_kwargs: fake_graph
+    )
+    monkeypatch.setattr(
+        pipeline, "_build_supabase_client", lambda *_args, **_kwargs: FakeGraphVectorSupabase(vector_rows)
+    )
+
+    rows = pipeline._retrieve_graph_candidates(
+        "which one talks about Alehin",
+        route_mode="explicit_compare",
+        access_token="token",
+        user_id="user-1",
+        priority_file_hashes=["A"],
+    )
+
+    assert len(rows) == 1
+    assert rows[0]["metadata"]["retrieval_branch"] == "graph_traversal"
+    assert rows[0]["metadata"]["graph_hit_depth"] >= 0
+
+
+def test_admin_promote_feedback_creates_eval_dataset(monkeypatch):
+    fake_service = FakeServiceSupabase()
+    fake_service.trace_rows.append(
+        {
+            "trace_id": "8f8c1f3f-bcb6-43a8-b10d-85f31a917111",
+            "question": "What is common between these two documents?",
+            "doc_diagnostics": [{"source": "BEYOND BOUNDS", "reason": "insufficient_coverage"}],
+            "failure_modes": ["unsupported_commonality"],
+            "answer_preview": "The documents both explore emotion.",
+            "review_state": "pending",
+        }
+    )
+    fake_service.feedback_rows.append(
+        {
+            "id": 7,
+            "trace_id": "8f8c1f3f-bcb6-43a8-b10d-85f31a917111",
+            "helpful": False,
+            "accepted": False,
+            "reason_code": "unsupported_commonality",
+            "correction_text": "Insufficient evidence for commonality.",
+            "promote_to_eval": True,
+            "review_state": "pending",
+        }
+    )
+
+    monkeypatch.setattr(admin, "_admin_client", lambda: fake_service)
+    monkeypatch.setenv("MASTER_ADMIN_KEY", "secret")
+
+    result = admin.promote_feedback_to_eval(7, x_admin_key="secret")
+
+    assert result["ok"] is True
+    assert len(fake_service.eval_rows) == 1
+    assert fake_service.eval_rows[0]["trace_id"] == "8f8c1f3f-bcb6-43a8-b10d-85f31a917111"
+    assert fake_service.trace_rows[0]["review_state"] == "promoted"
+    assert fake_service.feedback_rows[0]["review_state"] == "promoted"

tests/test_routing_stress_matrix.py

@@ -0,0 +1,98 @@
+from types import SimpleNamespace
+
+from backend.core import pipeline
+
+
+class FakeFilesTable:
+    def __init__(self, rows):
+        self.rows = rows
+        self.filters = {}
+
+    def select(self, *_args):
+        return self
+
+    def eq(self, key, value):
+        self.filters[key] = value
+        return self
+
+    def execute(self):
+        return SimpleNamespace(data=list(self.rows))
+
+
+class FakeRpc:
+    def __init__(self, supabase, params):
+        self.supabase = supabase
+        self.params = params
+
+    def execute(self):
+        self.supabase.rpc_calls.append(self.params)
+        # Always return a row so ambiguity code can compute file_scores
+        return SimpleNamespace(data=[{"combined_score": 0.2}])
+
+
+class FakeSupabase:
+    def __init__(self, rows):
+        self.rows = rows
+        self.rpc_calls = []
+
+    def table(self, _name: str):
+        return FakeFilesTable(self.rows)
+
+    def rpc(self, _name: str, params):
+        return FakeRpc(self, params)
+
+
+def test_stress_matrix_identity_queries_never_guess_in_multi_doc_all_scope(monkeypatch):
+    """
+    Invariant: if multiple docs exist and the user hasn't pinned a doc (category=All),
+    identity/page-scoped queries must force clarification instead of falling through.
+    """
+    fake = FakeSupabase(
+        rows=[
+            {"file_hash": "A", "filename": "Guide A.pdf"},
+            {"file_hash": "B", "filename": "Guide B.pdf"},
+            {"file_hash": "C", "filename": "Guide C.pdf"},
+        ]
+    )
+    monkeypatch.setattr(pipeline, "_build_supabase_client", lambda *_args, **_kwargs: fake)
+
+    identity_like_queries = [
+        "Whose guide is this?",
+        "What is the exact full title of this guide?",
+        "What exact wording on the cover shows this guide is personalized?",
+        "Summarize only the first page, not the whole guide.",
+        "Does this guide explicitly name a publisher on the opening pages? If not, say not stated.",
+        "Publisher on the opening pages?",
+        "Cover wording?",
+        "Page 1 summary only.",
+    ]
+    for q in identity_like_queries:
+        res = pipeline.check_query_ambiguity(q, access_token=None, category="All")
+        assert res["is_ambiguous"] is True, q
+        assert res["top_file_hash"] is None, q
+
+    # For identity/page-scoped safety, we should not do per-file scoring RPC calls.
+    assert fake.rpc_calls == []
+
+
+def test_stress_matrix_generic_queries_may_use_scoring_and_include_p_user_id(monkeypatch):
+    fake = FakeSupabase(
+        rows=[
+            {"file_hash": "A", "filename": "Doc A.pdf"},
+            {"file_hash": "B", "filename": "Doc B.pdf"},
+        ]
+    )
+    monkeypatch.setattr(pipeline, "_build_supabase_client", lambda *_args, **_kwargs: fake)
+
+    generic_queries = [
+        "summarize the document",
+        "give me an overview",
+        "explain what this is about",
+    ]
+    for q in generic_queries:
+        res = pipeline.check_query_ambiguity(q, access_token=None, category="All")
+        assert res["is_ambiguous"] in {True, False}
+
+    assert fake.rpc_calls, "Expected scoring calls for generic multi-doc queries"
+    assert all("p_user_id" in call for call in fake.rpc_calls)
+