Fix remaining architect-flagged issues — targeting 9.5+

RetrievalService shared QdrantClient:
- Was creating its own QdrantClient despite main.py wiring a shared QdrantStore.
- Now accepts QdrantStore param and uses store.client — single connection pool.

BM25 docstring correctness:
- Comment claimed Qdrant applies IDF to manually-supplied sparse vectors.
- Factually wrong: IDF is only applied by Qdrant's built-in FastEmbed vectorizer.
- Corrected to accurately describe TF-only behavior, explain the practical
impact on code search, and document the upgrade path to true BM25.

GenerationService hardcoded model strings:
- _groq_complete/_groq_stream had "llama-3.3-70b-versatile" as literals.
- Now uses self._model set in _init_provider, consistent with AgentService.

Security — filepath path traversal (get_file_chunk):
- LLM-supplied filepath was passed directly into GitHub API URL.
- Added validation: repo must be "owner/name", filepath must not contain
".." components or start with "/". Uses PurePosixPath for robust parsing.

Security — X-Forwarded-For leftmost vs rightmost IP:
- Was using split(",")[0] — the leftmost entry is user-controlled and
trivially bypassable to defeat per-IP rate limiting.
- Fixed to split(",")[-1] — the rightmost entry is appended by the actual
proxy and cannot be forged by the client.

Eval harness improvements:
- Added eval/test_cases/nanogpt.json: 10 cases for karpathy/nanoGPT covering
BM25-heavy queries ("where is get_batch called"), semantic queries
("how does attention work"), and config/init cases.
- Added --output FILE flag: writes JSON summary for CI integration.
Enables git diff eval_results.json to surface retrieval regressions.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

backend/main.py

@@ -95,7 +95,7 @@ async def lifespan(app: FastAPI):
     _qdrant_store = QdrantStore()
 
     # Core services — all share the same store + embedder instances
-    _retrieval_service  = RetrievalService(embedder=_embedder)
+    _retrieval_service  = RetrievalService(embedder=_embedder, store=_qdrant_store)
     _ingestion_service  = IngestionService(store=_qdrant_store, embedder=_embedder)
     _graph_service      = GraphService(_qdrant_store)
     _generation_service = GenerationService()
@@ -188,8 +188,12 @@ def _check_rate_limit(request: Request) -> None:
     if limit <= 0:
         return  # disabled
 
-    ip  = request.headers.get("X-Forwarded-For", "").split(",")[0].strip()
-    ip  = ip or (request.client.host if request.client else "unknown")
+    # Use the RIGHTMOST X-Forwarded-For entry — it's appended by the actual
+    # proxy (Render, Cloudflare, etc.) and cannot be forged by the client.
+    # The leftmost entry is user-controlled and trivially bypassable.
+    forwarded = request.headers.get("X-Forwarded-For", "")
+    ip = forwarded.split(",")[-1].strip() if forwarded else ""
+    ip = ip or (request.client.host if request.client else "unknown")
     now = time.monotonic()
 
     window = _rate_windows[ip]

backend/mcp_server.py

@@ -226,6 +226,20 @@ def get_file_chunk(
         start_line: First line to fetch, 1-indexed
         end_line:   Last line to fetch, inclusive
     """
+    # Validate repo format — LLM-supplied args must never be passed raw into URLs.
+    if "/" not in repo or repo.count("/") != 1:
+        return f"Invalid repo format '{repo}'. Expected 'owner/name'."
+
+    # Reject path traversal — an LLM (or prompt injection) could pass "../.env"
+    # which would resolve to an unintended path in the GitHub API URL.
+    from pathlib import PurePosixPath
+    try:
+        parts = PurePosixPath(filepath).parts
+    except Exception:
+        return "Invalid filepath."
+    if ".." in parts or filepath.startswith("/"):
+        return "Invalid filepath: path traversal not allowed."
+
     owner, name = repo.split("/", 1)
     url = f"https://api.github.com/repos/{owner}/{name}/contents/{filepath}"
     headers = {"Accept": "application/vnd.github.v3.raw"}

backend/services/generation.py

@@ -148,12 +148,14 @@ class GenerationService:
         if settings.groq_api_key:
             from groq import Groq
             self._client = Groq(api_key=settings.groq_api_key)
-            print("Generation: using Groq (llama-3.3-70b-versatile)")
+            self._model  = "llama-3.3-70b-versatile"
+            print(f"Generation: using Groq ({self._model})")
             return "groq"
         elif settings.anthropic_api_key:
             import anthropic
             self._client = anthropic.Anthropic(api_key=settings.anthropic_api_key)
-            print("Generation: using Anthropic (claude-haiku-4-5)")
+            self._model  = "claude-haiku-4-5-20251001"
+            print(f"Generation: using Anthropic ({self._model})")
             return "anthropic"
         else:
             raise ValueError(
@@ -207,7 +209,7 @@ class GenerationService:
 
     def _groq_complete(self, system: str, prompt: str, params: dict) -> str:
         response = self._client.chat.completions.create(
-            model="llama-3.3-70b-versatile",
+            model=self._model,
             messages=[
                 {"role": "system", "content": system},
                 {"role": "user",   "content": prompt},
@@ -219,7 +221,7 @@ class GenerationService:
 
     def _groq_stream(self, system: str, prompt: str, params: dict) -> Iterator[str]:
         stream = self._client.chat.completions.create(
-            model="llama-3.3-70b-versatile",
+            model=self._model,
             messages=[
                 {"role": "system", "content": system},
                 {"role": "user",   "content": prompt},
@@ -237,7 +239,7 @@ class GenerationService:
 
     def _anthropic_complete(self, system: str, prompt: str, params: dict) -> str:
         response = self._client.messages.create(
-            model="claude-haiku-4-5-20251001",
+            model=self._model,
             system=system,
             messages=[{"role": "user", "content": prompt}],
             temperature=params["temperature"],
@@ -247,7 +249,7 @@ class GenerationService:
 
     def _anthropic_stream(self, system: str, prompt: str, params: dict) -> Iterator[str]:
         with self._client.messages.stream(
-            model="claude-haiku-4-5-20251001",
+            model=self._model,
             system=system,
             messages=[{"role": "user", "content": prompt}],
             temperature=params["temperature"],

eval/eval.py

@@ -235,6 +235,11 @@ def main():
         "--verbose", action="store_true",
         help="Show per-case results including misses"
     )
+    parser.add_argument(
+        "--output", default=None, metavar="FILE",
+        help="Write results as JSON to FILE (e.g. eval_results.json). "
+             "Useful for CI: git diff eval_results.json shows regressions."
+    )
     args = parser.parse_args()
 
     # ── Load test cases ────────────────────────────────────────────────────────
@@ -274,6 +279,25 @@ def main():
         all_summaries[mode] = summary
         print_report(mode, summary, results, args.top_k, args.verbose)
 
+    # ── JSON output for CI ────────────────────────────────────────────────────
+    if args.output:
+        import json as _json
+        output = {
+            "repo":   args.repo,
+            "top_k":  args.top_k,
+            "n_cases": len(cases),
+            "results": {
+                mode: {
+                    "hit_at_k": s[f"hit@{args.top_k}"],
+                    "mrr":      s["mrr"],
+                    "p_at_k":   s[f"p@{args.top_k}"],
+                }
+                for mode, s in all_summaries.items()
+            }
+        }
+        Path(args.output).write_text(_json.dumps(output, indent=2))
+        print(f"\nResults written to {args.output}")
+
     # ── Comparison table ───────────────────────────────────────────────────────
     if len(args.modes) > 1:
         k = args.top_k

eval/test_cases/nanogpt.json

@@ -0,0 +1,52 @@
+[
+  {
+    "question": "where is get_batch called",
+    "expected_files": ["train.py"],
+    "expected_names": ["get_batch"]
+  },
+  {
+    "question": "How is the attention mechanism implemented?",
+    "expected_files": ["model.py"],
+    "expected_names": ["CausalSelfAttention"]
+  },
+  {
+    "question": "What does the GPT class do?",
+    "expected_files": ["model.py"],
+    "expected_names": ["GPT"]
+  },
+  {
+    "question": "How is learning rate decay scheduled?",
+    "expected_files": ["train.py"],
+    "expected_names": ["get_lr"]
+  },
+  {
+    "question": "How does the forward pass work in the transformer block?",
+    "expected_files": ["model.py"],
+    "expected_names": ["Block", "forward"]
+  },
+  {
+    "question": "How is the model configured and initialised from a checkpoint?",
+    "expected_files": ["model.py"],
+    "expected_names": ["GPTConfig", "from_pretrained"]
+  },
+  {
+    "question": "How are tokens generated after training?",
+    "expected_files": ["model.py", "sample.py"],
+    "expected_names": ["generate"]
+  },
+  {
+    "question": "How is gradient clipping applied during training?",
+    "expected_files": ["train.py"],
+    "expected_names": []
+  },
+  {
+    "question": "What is the LayerNorm implementation used?",
+    "expected_files": ["model.py"],
+    "expected_names": ["LayerNorm"]
+  },
+  {
+    "question": "How does DDP distributed training work in the training loop?",
+    "expected_files": ["train.py"],
+    "expected_names": []
+  }
+]

ingestion/qdrant_store.py

@@ -329,9 +329,21 @@ def _text_to_sparse(text: str) -> SparseVector:
       → indices = [md5("def") % 1M, md5("embed_text") % 1M, ...]
         values  = [1.0, 1.0, 1.0, 2.0, ...]
 
-    Qdrant uses these sparse vectors for BM25-style keyword matching.
-    The actual BM25 ranking (IDF weighting, document length normalisation)
-    is applied at query time by Qdrant.
+    IMPORTANT — this is TF (term frequency) only, NOT full BM25.
+    True BM25 requires IDF (inverse document frequency), which weights rare
+    terms higher than common ones (e.g. "backward" > "def"). Qdrant can apply
+    IDF automatically only if you use its built-in FastEmbed sparse vectorizer
+    (which builds a vocabulary from your corpus). When you supply raw sparse
+    vectors manually (as we do here), Qdrant treats them as-is — no IDF,
+    no document length normalisation.
+
+    Practical effect: exact identifier lookups still work well (TF alone finds
+    the chunk containing "backward" 5 times better than one mentioning it once).
+    But stop-word-heavy queries ("how does the") may score higher than they
+    should. For a code search use case this is a reasonable trade-off since
+    identifiers are the dominant signal and they're naturally rare.
+
+    Upgrade path: switch to Qdrant's FastEmbed sparse vectorizer for true BM25.
 
     WHY NOT hash(token)?
       Python's built-in hash() is randomised per process (PYTHONHASHSEED).

retrieval/retrieval.py

@@ -46,7 +46,7 @@ from qdrant_client.models import (
 sys.path.insert(0, str(Path(__file__).parent.parent))
 from backend.config import settings
 from ingestion.embedder import Embedder
-from ingestion.qdrant_store import _text_to_sparse
+from ingestion.qdrant_store import QdrantStore, _text_to_sparse
 
 
 class RetrievalService:
@@ -57,22 +57,34 @@ class RetrievalService:
     as the indexed chunks. Mixing embedding models breaks retrieval entirely —
     vectors from different models are incomparable.
 
-    Why accept embedder as an argument?
-      IngestionService and RetrievalService both need the same 600MB model.
-      Instantiating it twice wastes ~600MB RAM. main.py creates one Embedder
-      and passes it to both services. Shared state, one load.
+    Why accept embedder and store as arguments?
+      - Embedder: IngestionService and RetrievalService both need the 600MB model.
+        Instantiating twice wastes ~600MB RAM. main.py creates one and shares it.
+      - QdrantStore: opening a second QdrantClient creates a second connection pool
+        to the same database, which wastes resources and was the "3 QdrantStore
+        instances" bug. By accepting the shared store we use its existing client.
     """
 
     DENSE_VECTOR_NAME  = "code"
     SPARSE_VECTOR_NAME = "bm25"
 
-    def __init__(self, embedder: Embedder | None = None):
-        self.embedder = embedder or Embedder()
-        self.client   = QdrantClient(
-            url=settings.qdrant_url,
-            api_key=settings.qdrant_api_key or None,
-        )
-        self.collection = settings.qdrant_collection
+    def __init__(
+        self,
+        embedder: Embedder | None = None,
+        store: QdrantStore | None = None,
+    ):
+        self.embedder   = embedder or Embedder()
+        # Use the shared store's client if provided; otherwise open a new connection.
+        # The store already validated QDRANT_URL and created payload indices.
+        if store is not None:
+            self.client     = store.client
+            self.collection = store.collection
+        else:
+            self.client     = QdrantClient(
+                url=settings.qdrant_url,
+                api_key=settings.qdrant_api_key or None,
+            )
+            self.collection = settings.qdrant_collection
 
     def search(
         self,