feat(rag): ingest CLI (markdown/PDF → chunks → FAISS) + sample KB fixtures

src/rag/ingest.py

@@ -0,0 +1,85 @@
+"""Walk a knowledge-base directory, chunk each file, embed, persist FAISS index.
+
+CLI entry point: `python -m src.rag.ingest [<input_dir> [<output_dir>]]`.
+Defaults: input=`data/knowledge_base/`, output=`data/processed/faiss_index/`.
+
+Supported file types: `.md`, `.txt`, `.pdf`. Other extensions are ignored
+with a logged WARNING.
+"""
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+from src.core.logger import get_logger
+from src.rag.chunker import chunk_text
+from src.rag.embed import EMBEDDING_DIM, Embedder
+from src.rag.store import FAISSStore
+
+logger = get_logger(__name__)
+
+
+_DEFAULT_INPUT = Path("data/knowledge_base")
+_DEFAULT_OUTPUT = Path("data/processed/faiss_index")
+_SUPPORTED = {".md", ".txt", ".pdf"}
+
+
+def _read_pdf(path: Path) -> str:
+    from pypdf import PdfReader
+    reader = PdfReader(str(path))
+    return "\n\n".join(page.extract_text() or "" for page in reader.pages)
+
+
+def _read_file(path: Path) -> str:
+    suffix = path.suffix.lower()
+    if suffix == ".pdf":
+        return _read_pdf(path)
+    return path.read_text(encoding="utf-8", errors="replace")
+
+
+def ingest_directory(input_dir: Path, output_dir: Path) -> int:
+    """Ingest every supported file in `input_dir` into a FAISS index at `output_dir`.
+
+    Returns the total number of chunks indexed.
+    """
+    input_dir = Path(input_dir)
+    output_dir = Path(output_dir)
+
+    files = sorted(p for p in input_dir.rglob("*") if p.suffix.lower() in _SUPPORTED)
+    logger.info("Ingesting %d file(s) from %s", len(files), input_dir)
+
+    all_chunks: list[dict] = []
+    for path in files:
+        try:
+            text = _read_file(path)
+        except Exception as e:
+            logger.warning("Skipping %s (read failed: %s)", path, e)
+            continue
+        for i, ch in enumerate(chunk_text(text)):
+            all_chunks.append({
+                "text": ch,
+                "source": str(path.relative_to(input_dir)),
+                "chunk_index": i,
+            })
+
+    store = FAISSStore(dim=EMBEDDING_DIM)
+    if all_chunks:
+        embedder = Embedder()
+        vectors = embedder.encode([c["text"] for c in all_chunks])
+        store.add(vectors, all_chunks)
+
+    store.save(output_dir)
+    logger.info("Indexed %d chunk(s) → %s", len(all_chunks), output_dir)
+    return len(all_chunks)
+
+
+def main() -> None:
+    args = sys.argv[1:]
+    inp = Path(args[0]) if len(args) >= 1 else _DEFAULT_INPUT
+    out = Path(args[1]) if len(args) >= 2 else _DEFAULT_OUTPUT
+    n = ingest_directory(inp, out)
+    print(f"Indexed {n} chunks into {out}")
+
+
+if __name__ == "__main__":
+    main()

tests/fixtures/kb_sample/combat_harmonization_primer.md

@@ -0,0 +1,27 @@
+# ComBat Harmonization for Multi-Site Neuroimaging
+
+ComBat (Johnson et al. 2007, adapted to MRI by Fortin et al. 2017, 2018)
+is the de-facto standard for removing scanner / acquisition-site bias
+from multi-center neuroimaging studies.
+
+## How it works
+
+ComBat models per-site location (mean) and scale (variance) parameters
+using an empirical-Bayes hierarchical framework. It estimates these
+parameters jointly across all sites and shrinks them toward a global
+prior — small-N sites are pulled toward the global mean, preventing
+overfitting.
+
+## Site-gap reduction
+
+A typical demonstration: the per-site mean of a hippocampus volume
+feature can vary by 5+ standard deviations across hospitals. ComBat
+typically collapses this gap to <0.005 — a 1000x+ reduction — while
+preserving within-site biological variance (age, sex, diagnosis).
+
+## When it fails
+
+ComBat requires at least 2 sites with overlapping covariate
+distributions. Single-site data, or sites with completely disjoint
+populations (e.g., one site only-pediatric, another only-elderly),
+produce unreliable harmonization.

tests/fixtures/kb_sample/lipinski_rule_of_five.md

@@ -0,0 +1,30 @@
+# Lipinski's Rule of Five — BBB Permeability Heuristic
+
+Lipinski's Rule of Five (Lipinski 1997, 2001) is the foundational
+medicinal-chemistry rule for predicting whether a small molecule will
+cross the blood-brain barrier (BBB) by passive diffusion.
+
+## The four criteria
+
+A molecule is likely BBB-permeable if it satisfies all four:
+
+1. Molecular weight (MW) <= 500 Daltons
+2. Octanol-water partition coefficient (logP) <= 5
+3. Hydrogen-bond donors <= 5
+4. Hydrogen-bond acceptors <= 10
+
+Molecules violating two or more criteria are typically poorly absorbed
+or impermeant.
+
+## Why ethanol crosses
+
+Ethanol (CCO) has MW=46 Da, logP=-0.31, 1 H-bond donor, 1 H-bond
+acceptor — well within all four thresholds. This explains its rapid
+CNS penetration despite hydrophilicity.
+
+## SHAP attribution interpretation
+
+When a Random Forest BBB classifier flags Morgan fingerprint bits with
+positive SHAP values toward a "permeable" label, the bit usually
+corresponds to a small lipophilic substructure (CH3-, -OCH3-, aromatic
+ring) consistent with Lipinski compliance.

tests/fixtures/kb_sample/mne_ica_basics.md

@@ -0,0 +1,29 @@
+# MNE-Python ICA for EEG Artifact Removal
+
+Independent Component Analysis (ICA, Hyvärinen 1999) decomposes a
+multi-channel EEG recording into statistically independent source
+components. It is the de-facto method for removing eye-blink and
+heartbeat artifacts before downstream analysis.
+
+## Why ICA, not PCA
+
+PCA decomposes signals into orthogonal components — but neural sources
+are not orthogonal in scalp space, they are statistically independent.
+ICA's independence assumption matches the physics: the eye, the heart,
+and cortical sources fire on uncorrelated schedules.
+
+## The standard workflow
+
+1. Bandpass the raw recording at 0.5-40 Hz to remove DC drift and line
+   noise (50/60 Hz).
+2. Fit ICA with N components (typically 15-30, less than channel count).
+3. Identify artifact components by correlating each ICA source with the
+   EOG (eye) channel; reject components with |correlation| > 0.5.
+4. Reconstruct the cleaned signal by zeroing out the rejected
+   components and inverse-transforming.
+
+## Quality check
+
+Post-ICA, the EOG channel should show minimal residual correlation
+with frontal channels (Fp1/Fp2). If it doesn't, the ICA fit was likely
+unstable — re-run with a different random seed or more components.

tests/rag/test_ingest.py

@@ -0,0 +1,40 @@
+"""Tests for src.rag.ingest — walk a directory, chunk, embed, persist."""
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+import pytest
+
+from src.rag.ingest import ingest_directory
+from src.rag.store import FAISSStore
+
+
+_FIXTURE_KB = Path(__file__).parent.parent / "fixtures" / "kb_sample"
+
+
+class TestIngestDirectory:
+    def test_ingests_markdown_files(self, tmp_path: Path) -> None:
+        out_dir = tmp_path / "idx"
+        n = ingest_directory(_FIXTURE_KB, out_dir)
+        assert n > 0  # at least one chunk per fixture file
+        assert (out_dir / "index.bin").exists()
+        assert (out_dir / "chunks.json").exists()
+
+    def test_loaded_store_is_searchable(self, tmp_path: Path) -> None:
+        out_dir = tmp_path / "idx"
+        ingest_directory(_FIXTURE_KB, out_dir)
+        from src.rag.embed import EMBEDDING_DIM
+        store = FAISSStore.load(out_dir, dim=EMBEDDING_DIM)
+        assert len(store) > 0
+        # chunks have source metadata
+        assert all("source" in c for c in store._chunks)
+        assert all("text" in c for c in store._chunks)
+
+    def test_empty_directory_creates_empty_index(self, tmp_path: Path) -> None:
+        empty = tmp_path / "empty_kb"
+        empty.mkdir()
+        out_dir = tmp_path / "idx"
+        n = ingest_directory(empty, out_dir)
+        assert n == 0
+        assert (out_dir / "index.bin").exists()