diff --git "a/HALT_RAG_Demo.ipynb" "b/HALT_RAG_Demo.ipynb" new file mode 100644--- /dev/null +++ "b/HALT_RAG_Demo.ipynb" @@ -0,0 +1,2230 @@ +{ + "nbformat": 4, + "nbformat_minor": 5, + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10.12", + "mimetype": "text/x-python", + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "pygments_lexer": "ipython3" + }, + "accelerator": "GPU", + "gpuClass": "premium" + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# \ud83d\udee1\ufe0f HALT-RAG: Hallucination-Aware Retrieval-Augmented Generation\n", + "\n", + "A complete research-style demo implementing:\n", + "- **Experimental retrieval strategies** (Hybrid BM25+FAISS, Dense-only, Two-stage rerank)\n", + "- **Lightweight agent workflow** (Planner \u2192 Executor \u2192 Critic \u2192 Logger)\n", + "- **Hallucination detection** with classification\n", + "- **Structured logging**\n", + "- **Evaluation + visualization**\n", + "\n", + "**Runtime:** Google Colab Pro (A100 GPU) \n", + "**Author:** Graduate AI Systems Project \n", + "**License:** MIT \n", + "\n", + "> \u26a0\ufe0f This uses a **SYNTHETIC** dataset for evaluation purposes only." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 1: Setup & Installation\n", + "\n", + "Install required packages and verify GPU availability." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# \u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n", + "# \u2551 SECTION 1: SETUP & INSTALLATION \u2551\n", + "# \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n", + "\n", + "!pip install -q sentence-transformers faiss-cpu rank-bm25 transformers accelerate matplotlib pandas numpy\n", + "\n", + "import time\n", + "import json\n", + "import re\n", + "import warnings\n", + "from dataclasses import dataclass, field, asdict\n", + "from typing import List, Dict, Tuple, Optional, Any\n", + "from collections import Counter\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import torch\n", + "import faiss\n", + "from rank_bm25 import BM25Okapi\n", + "from sentence_transformers import SentenceTransformer\n", + "from transformers import pipeline, AutoModelForCausalLM, AutoTokenizer\n", + "\n", + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "print(\"=\" * 70)\n", + "print(\"HALT-RAG: Hallucination-Aware Retrieval-Augmented Generation\")\n", + "print(\"=\" * 70)\n", + "print(f\"PyTorch version: {torch.__version__}\")\n", + "print(f\"CUDA available: {torch.cuda.is_available()}\")\n", + "if torch.cuda.is_available():\n", + " print(f\"GPU: {torch.cuda.get_device_name(0)}\")\n", + "print()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 2: Synthetic Evaluation Dataset\n", + "\n", + "Create a synthetic evaluation dataset with 55 test cases across multiple domains.\n", + "\n", + "Each test case includes: `id`, `question`, `expected_answer`, `relevant_context`, `domain`, `difficulty`, `expected_hallucination_type`\n", + "\n", + "> **NOTE:** This dataset is SYNTHETIC \u2014 created for evaluation purposes only." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "print(\"=\" * 70)\n", + "print(\"SECTION 2: Creating Synthetic Evaluation Dataset\")\n", + "print(\"=\" * 70)\n", + "print(\"NOTE: This dataset is SYNTHETIC \u2014 created for evaluation purposes only.\")\n", + "print()\n", + "\n", + "\n", + "def create_synthetic_dataset() -> List[Dict]:\n", + " \"\"\"\n", + " Create a synthetic evaluation dataset with 50+ test cases.\n", + " Each case is designed to test different aspects of RAG + hallucination detection.\n", + "\n", + " Domains: Science, History, Technology, Medicine, Geography\n", + " Difficulties: easy, medium, hard\n", + " Expected hallucination types: factual, contextual, reasoning, faithful\n", + " \"\"\"\n", + " dataset = [\n", + " # === SCIENCE DOMAIN (10 cases) ===\n", + " {\n", + " \"id\": \"SCI-001\",\n", + " \"question\": \"What is the speed of light in a vacuum?\",\n", + " \"expected_answer\": \"The speed of light in a vacuum is approximately 299,792,458 meters per second.\",\n", + " \"relevant_context\": \"Light travels at a constant speed in a vacuum, measured precisely at 299,792,458 meters per second (approximately 3 \u00d7 10^8 m/s). This is denoted by the constant 'c' and is a fundamental physical constant.\",\n", + " \"domain\": \"science\",\n", + " \"difficulty\": \"easy\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"SCI-002\",\n", + " \"question\": \"What causes the aurora borealis?\",\n", + " \"expected_answer\": \"The aurora borealis is caused by charged particles from the sun interacting with gases in Earth's atmosphere.\",\n", + " \"relevant_context\": \"The aurora borealis (Northern Lights) occurs when charged particles from the solar wind interact with gases in Earth's upper atmosphere. These particles are guided by Earth's magnetic field toward the poles, where they collide with oxygen and nitrogen atoms, causing them to emit light.\",\n", + " \"domain\": \"science\",\n", + " \"difficulty\": \"medium\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"SCI-003\",\n", + " \"question\": \"How does photosynthesis convert sunlight into energy?\",\n", + " \"expected_answer\": \"Photosynthesis uses chlorophyll to absorb sunlight, converting CO2 and water into glucose and oxygen through light-dependent and light-independent reactions.\",\n", + " \"relevant_context\": \"Photosynthesis occurs in two stages: light-dependent reactions in the thylakoid membranes capture solar energy using chlorophyll pigments, splitting water molecules and generating ATP and NADPH. The Calvin cycle (light-independent reactions) in the stroma then uses these energy carriers to fix CO2 into glucose (C6H12O6), releasing oxygen as a byproduct.\",\n", + " \"domain\": \"science\",\n", + " \"difficulty\": \"medium\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"SCI-004\",\n", + " \"question\": \"What is the Heisenberg uncertainty principle?\",\n", + " \"expected_answer\": \"The Heisenberg uncertainty principle states that it is impossible to simultaneously know both the exact position and exact momentum of a particle.\",\n", + " \"relevant_context\": \"The Heisenberg uncertainty principle, formulated by Werner Heisenberg in 1927, establishes a fundamental limit in quantum mechanics. It states that the more precisely the position of a particle is determined, the less precisely its momentum can be known, and vice versa. Mathematically: \u0394x\u00b7\u0394p \u2265 \u210f/2.\",\n", + " \"domain\": \"science\",\n", + " \"difficulty\": \"hard\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"SCI-005\",\n", + " \"question\": \"What temperature does water boil at on Mars?\",\n", + " \"expected_answer\": \"Water boils at a much lower temperature on Mars due to the very low atmospheric pressure, roughly around 0-10\u00b0C depending on conditions.\",\n", + " \"relevant_context\": \"Mars has an atmospheric pressure of about 610 Pascals (0.6% of Earth's sea level pressure). Due to this extremely low pressure, the boiling point of water on Mars is significantly reduced. At typical Martian surface pressures, water would boil at approximately 0-2\u00b0C rather than 100\u00b0C as on Earth at sea level.\",\n", + " \"domain\": \"science\",\n", + " \"difficulty\": \"hard\",\n", + " \"expected_hallucination_type\": \"contextual\",\n", + " },\n", + " {\n", + " \"id\": \"SCI-006\",\n", + " \"question\": \"What is CRISPR-Cas9?\",\n", + " \"expected_answer\": \"CRISPR-Cas9 is a gene-editing technology that allows precise modification of DNA sequences in living organisms.\",\n", + " \"relevant_context\": \"CRISPR-Cas9 is a revolutionary gene-editing tool adapted from a natural defense mechanism in bacteria. The system uses a guide RNA to direct the Cas9 enzyme to a specific location in the genome, where it makes a precise cut in the DNA double helix. This allows researchers to delete, modify, or insert genetic material at targeted locations.\",\n", + " \"domain\": \"science\",\n", + " \"difficulty\": \"medium\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"SCI-007\",\n", + " \"question\": \"How do black holes form?\",\n", + " \"expected_answer\": \"Black holes form when massive stars exhaust their nuclear fuel and collapse under their own gravity.\",\n", + " \"relevant_context\": \"Stellar black holes form through gravitational collapse when a massive star (typically >25 solar masses) exhausts its nuclear fuel. Without the outward radiation pressure to counteract gravity, the core collapses. If the remaining mass exceeds the Tolman-Oppenheimer-Volkoff limit (~2-3 solar masses), neutron degeneracy pressure cannot halt the collapse, and a black hole forms.\",\n", + " \"domain\": \"science\",\n", + " \"difficulty\": \"medium\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"SCI-008\",\n", + " \"question\": \"What is dark matter made of?\",\n", + " \"expected_answer\": \"The composition of dark matter is unknown. Leading candidates include WIMPs and axions, but it has not been directly detected.\",\n", + " \"relevant_context\": \"Dark matter constitutes approximately 27% of the universe's mass-energy content, yet its composition remains one of physics' greatest mysteries. It does not emit, absorb, or reflect electromagnetic radiation. Leading theoretical candidates include Weakly Interacting Massive Particles (WIMPs), axions, and sterile neutrinos. Despite numerous detection experiments, dark matter has not been directly observed.\",\n", + " \"domain\": \"science\",\n", + " \"difficulty\": \"hard\",\n", + " \"expected_hallucination_type\": \"reasoning\",\n", + " },\n", + " {\n", + " \"id\": \"SCI-009\",\n", + " \"question\": \"What is the half-life of carbon-14?\",\n", + " \"expected_answer\": \"The half-life of carbon-14 is approximately 5,730 years.\",\n", + " \"relevant_context\": \"Carbon-14 (14C) is a radioactive isotope of carbon with a half-life of 5,730 \u00b1 40 years. It is continuously produced in the upper atmosphere by cosmic ray bombardment of nitrogen-14. This predictable decay rate makes it invaluable for radiocarbon dating of organic materials up to about 50,000 years old.\",\n", + " \"domain\": \"science\",\n", + " \"difficulty\": \"easy\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"SCI-010\",\n", + " \"question\": \"Why is the sky blue?\",\n", + " \"expected_answer\": \"The sky appears blue because of Rayleigh scattering \u2014 shorter blue wavelengths of sunlight are scattered more than longer wavelengths by atmospheric molecules.\",\n", + " \"relevant_context\": \"The blue color of the sky is caused by Rayleigh scattering. When sunlight enters Earth's atmosphere, it collides with gas molecules. Since the scattering intensity is inversely proportional to the fourth power of wavelength (1/\u03bb\u2074), shorter wavelengths (blue/violet) scatter much more than longer wavelengths (red/orange). Our eyes are more sensitive to blue than violet, so we perceive the sky as blue.\",\n", + " \"domain\": \"science\",\n", + " \"difficulty\": \"easy\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " # === HISTORY DOMAIN (10 cases) ===\n", + " {\n", + " \"id\": \"HIS-001\",\n", + " \"question\": \"When did World War II end?\",\n", + " \"expected_answer\": \"World War II ended in 1945, with Germany surrendering in May and Japan in August/September.\",\n", + " \"relevant_context\": \"World War II ended in 1945. Germany surrendered unconditionally on May 8, 1945 (V-E Day). Japan announced its surrender on August 15, 1945 (V-J Day), following the atomic bombings of Hiroshima and Nagasaki. The formal surrender was signed on September 2, 1945, aboard the USS Missouri in Tokyo Bay.\",\n", + " \"domain\": \"history\",\n", + " \"difficulty\": \"easy\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"HIS-002\",\n", + " \"question\": \"What caused the fall of the Roman Empire?\",\n", + " \"expected_answer\": \"The fall of the Roman Empire was caused by multiple factors including economic troubles, military overextension, barbarian invasions, and political instability.\",\n", + " \"relevant_context\": \"Historians attribute the fall of the Western Roman Empire (476 CE) to a complex combination of factors: severe economic troubles and overtaxation, military overextension and reliance on mercenaries, repeated barbarian invasions (Visigoths, Vandals, Huns), political instability with rapid emperor turnover, administrative division, and loss of traditional civic values.\",\n", + " \"domain\": \"history\",\n", + " \"difficulty\": \"medium\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"HIS-003\",\n", + " \"question\": \"Who built the Great Wall of China?\",\n", + " \"expected_answer\": \"The Great Wall was built over many centuries by multiple Chinese dynasties, with the most well-known sections constructed during the Ming Dynasty (1368-1644).\",\n", + " \"relevant_context\": \"The Great Wall of China was not built by a single ruler but constructed and reconstructed over approximately 2,000 years by multiple dynasties. The earliest walls date to the 7th century BCE. Emperor Qin Shi Huang connected existing walls around 221 BCE. The most recognizable sections were built during the Ming Dynasty (1368-1644 CE), stretching over 8,850 kilometers.\",\n", + " \"domain\": \"history\",\n", + " \"difficulty\": \"medium\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"HIS-004\",\n", + " \"question\": \"What was the significance of the Rosetta Stone?\",\n", + " \"expected_answer\": \"The Rosetta Stone was key to deciphering Egyptian hieroglyphics because it contained the same text in three scripts: hieroglyphic, Demotic, and Greek.\",\n", + " \"relevant_context\": \"The Rosetta Stone, discovered in 1799 by French soldiers in Egypt, contains a decree issued in 196 BCE written in three scripts: Ancient Egyptian hieroglyphics, Demotic script, and Ancient Greek. Since scholars could read Greek, they could use it as a reference to decode the hieroglyphics. Jean-Fran\u00e7ois Champollion finally deciphered the hieroglyphics in 1822.\",\n", + " \"domain\": \"history\",\n", + " \"difficulty\": \"medium\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"HIS-005\",\n", + " \"question\": \"What year did the first humans land on the Moon?\",\n", + " \"expected_answer\": \"The first humans landed on the Moon in 1969 during the Apollo 11 mission.\",\n", + " \"relevant_context\": \"On July 20, 1969, NASA's Apollo 11 mission successfully landed the first humans on the Moon. Astronauts Neil Armstrong and Buzz Aldrin walked on the lunar surface while Michael Collins orbited above in the command module. Armstrong's famous words upon stepping onto the surface were 'That's one small step for man, one giant leap for mankind.'\",\n", + " \"domain\": \"history\",\n", + " \"difficulty\": \"easy\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"HIS-006\",\n", + " \"question\": \"What were the main causes of World War I?\",\n", + " \"expected_answer\": \"The main causes of WWI include militarism, alliances, imperialism, nationalism, and the assassination of Archduke Franz Ferdinand.\",\n", + " \"relevant_context\": \"The causes of World War I are often summarized by the acronym MANIA: Militarism (arms race between European powers), Alliances (complex treaty obligations), Nationalism (ethnic tensions especially in the Balkans), Imperialism (competition for colonies), and the immediate trigger: the Assassination of Archduke Franz Ferdinand of Austria-Hungary on June 28, 1914, in Sarajevo.\",\n", + " \"domain\": \"history\",\n", + " \"difficulty\": \"medium\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"HIS-007\",\n", + " \"question\": \"How did the Black Death spread across Europe?\",\n", + " \"expected_answer\": \"The Black Death spread primarily through fleas on rats, traveling along trade routes from Central Asia to Europe via merchant ships.\",\n", + " \"relevant_context\": \"The Black Death (1347-1351) spread from Central Asia along the Silk Road trade routes. It reached Europe via merchant ships, notably arriving at the Sicilian port of Messina in October 1347. The plague was caused by Yersinia pestis bacteria, primarily transmitted through bites of infected fleas living on black rats. Crowded medieval cities with poor sanitation facilitated rapid spread, killing 30-60% of Europe's population.\",\n", + " \"domain\": \"history\",\n", + " \"difficulty\": \"medium\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"HIS-008\",\n", + " \"question\": \"What was the exact population of ancient Rome at its peak?\",\n", + " \"expected_answer\": \"The exact population is uncertain, but estimates suggest Rome had approximately 1 million inhabitants at its peak around 100-200 CE.\",\n", + " \"relevant_context\": \"The population of ancient Rome at its peak is debated among historians. Most estimates suggest the city housed between 800,000 and 1.2 million people during the 1st-2nd centuries CE. Census records exist but are incomplete and their interpretation is contested. The broader Roman Empire may have contained 55-70 million people at its height.\",\n", + " \"domain\": \"history\",\n", + " \"difficulty\": \"hard\",\n", + " \"expected_hallucination_type\": \"reasoning\",\n", + " },\n", + " {\n", + " \"id\": \"HIS-009\",\n", + " \"question\": \"Who invented the printing press?\",\n", + " \"expected_answer\": \"Johannes Gutenberg invented the movable-type printing press around 1440 in Mainz, Germany.\",\n", + " \"relevant_context\": \"Johannes Gutenberg is credited with inventing the movable-type printing press in Europe around 1440 in Mainz, Germany. His key innovation was the combination of a hand mold for casting metal type, oil-based ink, and a wooden press adapted from wine/olive presses. His most famous work, the Gutenberg Bible (1455), demonstrated the technology's potential. Note: movable type existed earlier in China (Bi Sheng, 1040 CE) and Korea.\",\n", + " \"domain\": \"history\",\n", + " \"difficulty\": \"easy\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"HIS-010\",\n", + " \"question\": \"What happened at the Battle of Thermopylae?\",\n", + " \"expected_answer\": \"At Thermopylae in 480 BCE, a small Greek force led by King Leonidas and 300 Spartans held the narrow pass against the massive Persian army of Xerxes for three days.\",\n", + " \"relevant_context\": \"The Battle of Thermopylae (480 BCE) was a famous last stand. King Leonidas I of Sparta, with approximately 300 Spartans and several thousand other Greek allies (estimates of 5,000-7,000 total), held the narrow coastal pass of Thermopylae against Xerxes I's Persian army (estimated 70,000-300,000). They held for three days before being outflanked via a mountain path revealed by a local traitor named Ephialtes.\",\n", + " \"domain\": \"history\",\n", + " \"difficulty\": \"hard\",\n", + " \"expected_hallucination_type\": \"contextual\",\n", + " },\n", + " # === TECHNOLOGY DOMAIN (10 cases) ===\n", + " {\n", + " \"id\": \"TECH-001\",\n", + " \"question\": \"What is a transformer model in machine learning?\",\n", + " \"expected_answer\": \"A transformer is a neural network architecture that uses self-attention mechanisms to process sequential data in parallel, introduced in the 'Attention Is All You Need' paper.\",\n", + " \"relevant_context\": \"The Transformer architecture was introduced in the 2017 paper 'Attention Is All You Need' by Vaswani et al. at Google. It uses self-attention mechanisms to process all positions in a sequence simultaneously (in parallel), unlike recurrent models (RNNs/LSTMs) that process sequentially. Key components include multi-head attention, positional encoding, and feed-forward layers. Transformers form the basis of models like BERT, GPT, and T5.\",\n", + " \"domain\": \"technology\",\n", + " \"difficulty\": \"medium\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"TECH-002\",\n", + " \"question\": \"How does blockchain work?\",\n", + " \"expected_answer\": \"Blockchain is a distributed ledger that records transactions in linked, cryptographically secured blocks. Each block contains a hash of the previous block, creating an immutable chain.\",\n", + " \"relevant_context\": \"A blockchain is a decentralized, distributed digital ledger. Transactions are grouped into blocks, each containing: a cryptographic hash of the previous block, a timestamp, and transaction data. This linking creates a chain where altering any block would invalidate all subsequent blocks. Consensus mechanisms (Proof of Work, Proof of Stake) ensure agreement among network participants without a central authority.\",\n", + " \"domain\": \"technology\",\n", + " \"difficulty\": \"medium\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"TECH-003\",\n", + " \"question\": \"What is quantum computing?\",\n", + " \"expected_answer\": \"Quantum computing uses quantum mechanical phenomena like superposition and entanglement to perform computations that are intractable for classical computers.\",\n", + " \"relevant_context\": \"Quantum computing harnesses quantum mechanical phenomena to process information. Unlike classical bits (0 or 1), quantum bits (qubits) can exist in superposition (both states simultaneously). Entanglement allows correlated qubits to be linked regardless of distance. These properties enable quantum computers to solve certain problems exponentially faster than classical computers, such as factoring large numbers (Shor's algorithm) and searching unsorted databases (Grover's algorithm).\",\n", + " \"domain\": \"technology\",\n", + " \"difficulty\": \"hard\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"TECH-004\",\n", + " \"question\": \"When was the first iPhone released?\",\n", + " \"expected_answer\": \"The first iPhone was released on June 29, 2007.\",\n", + " \"relevant_context\": \"The original iPhone was announced by Steve Jobs on January 9, 2007, at the Macworld Conference & Expo, and released on June 29, 2007. It featured a 3.5-inch touchscreen, 2-megapixel camera, and ran iPhone OS 1.0. It was initially exclusive to AT&T in the United States and priced at $499 for the 4GB model and $599 for 8GB.\",\n", + " \"domain\": \"technology\",\n", + " \"difficulty\": \"easy\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"TECH-005\",\n", + " \"question\": \"What is the difference between supervised and unsupervised learning?\",\n", + " \"expected_answer\": \"Supervised learning uses labeled data to train models to predict outcomes, while unsupervised learning finds patterns in unlabeled data without predefined targets.\",\n", + " \"relevant_context\": \"In machine learning, supervised learning trains models on labeled datasets where each example has an input-output pair. The model learns to map inputs to correct outputs (classification, regression). Unsupervised learning works with unlabeled data, discovering hidden patterns, groupings, or structure (clustering, dimensionality reduction, anomaly detection). Semi-supervised learning combines both approaches.\",\n", + " \"domain\": \"technology\",\n", + " \"difficulty\": \"easy\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"TECH-006\",\n", + " \"question\": \"How does GPT-4 compare to GPT-3 in terms of parameters?\",\n", + " \"expected_answer\": \"The exact parameter count of GPT-4 has not been officially disclosed by OpenAI, unlike GPT-3 which has 175 billion parameters.\",\n", + " \"relevant_context\": \"GPT-3, released in 2020, contains 175 billion parameters. OpenAI has not officially disclosed the parameter count of GPT-4 (released March 2023). Various rumors and leaks suggest it may use a Mixture of Experts architecture with approximately 1.7 trillion total parameters across 16 experts, but this has not been confirmed by OpenAI. The company has moved away from disclosing model sizes.\",\n", + " \"domain\": \"technology\",\n", + " \"difficulty\": \"hard\",\n", + " \"expected_hallucination_type\": \"reasoning\",\n", + " },\n", + " {\n", + " \"id\": \"TECH-007\",\n", + " \"question\": \"What programming language is Python named after?\",\n", + " \"expected_answer\": \"Python is named after the BBC comedy series 'Monty Python's Flying Circus', not the snake.\",\n", + " \"relevant_context\": \"Python was created by Guido van Rossum and first released in 1991. The language is named after the British comedy group Monty Python (specifically their show 'Monty Python's Flying Circus'), not after the snake. Van Rossum was reading scripts from the show while developing the language and wanted a short, unique, and slightly mysterious name.\",\n", + " \"domain\": \"technology\",\n", + " \"difficulty\": \"easy\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"TECH-008\",\n", + " \"question\": \"What is federated learning?\",\n", + " \"expected_answer\": \"Federated learning is a machine learning approach where models are trained across decentralized devices holding local data, without exchanging raw data.\",\n", + " \"relevant_context\": \"Federated learning is a distributed machine learning approach proposed by Google in 2016. Instead of centralizing training data, the model is sent to devices (phones, hospitals, etc.) where it trains on local data. Only model updates (gradients) are sent back to a central server for aggregation. This preserves data privacy as raw data never leaves the device. Challenges include non-IID data distributions and communication efficiency.\",\n", + " \"domain\": \"technology\",\n", + " \"difficulty\": \"hard\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"TECH-009\",\n", + " \"question\": \"How many transistors are in a modern CPU?\",\n", + " \"expected_answer\": \"Modern CPUs contain tens of billions of transistors. For example, Apple's M2 Ultra has about 134 billion transistors.\",\n", + " \"relevant_context\": \"Modern processors contain billions of transistors. As of 2023-2024, high-end examples include: Apple M2 Ultra (~134 billion transistors), AMD EPYC Genoa (~90 billion), Intel's Ponte Vecchio GPU (~100 billion). Consumer CPUs like AMD Ryzen 9 7950X contain about 13 billion transistors. Transistor counts have followed Moore's Law for decades, roughly doubling every 2 years.\",\n", + " \"domain\": \"technology\",\n", + " \"difficulty\": \"medium\",\n", + " \"expected_hallucination_type\": \"factual\",\n", + " },\n", + " {\n", + " \"id\": \"TECH-010\",\n", + " \"question\": \"What is the halting problem?\",\n", + " \"expected_answer\": \"The halting problem proves that no general algorithm can determine whether an arbitrary program will eventually halt or run forever.\",\n", + " \"relevant_context\": \"The halting problem, proven undecidable by Alan Turing in 1936, states that no general algorithm can determine, for all possible program-input pairs, whether the program will eventually halt (finish) or continue to run forever. This was proven by contradiction: assuming such an algorithm exists leads to a logical paradox. It establishes fundamental limits on what computers can decide.\",\n", + " \"domain\": \"technology\",\n", + " \"difficulty\": \"hard\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " # === MEDICINE DOMAIN (10 cases) ===\n", + " {\n", + " \"id\": \"MED-001\",\n", + " \"question\": \"What is the function of insulin in the human body?\",\n", + " \"expected_answer\": \"Insulin regulates blood glucose levels by enabling cells to absorb glucose from the bloodstream for energy or storage.\",\n", + " \"relevant_context\": \"Insulin is a peptide hormone produced by beta cells in the pancreatic islets of Langerhans. Its primary function is to regulate blood glucose levels. When blood sugar rises (e.g., after eating), insulin signals cells (especially muscle, fat, and liver cells) to absorb glucose from the bloodstream. It promotes glycogen synthesis in the liver and fat storage in adipose tissue, effectively lowering blood sugar levels.\",\n", + " \"domain\": \"medicine\",\n", + " \"difficulty\": \"easy\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"MED-002\",\n", + " \"question\": \"How do mRNA vaccines work?\",\n", + " \"expected_answer\": \"mRNA vaccines deliver synthetic mRNA that instructs cells to produce a harmless piece of the target virus (like the spike protein), triggering an immune response.\",\n", + " \"relevant_context\": \"mRNA vaccines contain synthetic messenger RNA encoding instructions for producing a specific viral protein (e.g., SARS-CoV-2 spike protein). Once injected, the mRNA enters cells and is translated by ribosomes into the target protein. The immune system recognizes this protein as foreign, producing antibodies and training T-cells. The mRNA is naturally degraded within days and does not alter DNA, which remains in the nucleus.\",\n", + " \"domain\": \"medicine\",\n", + " \"difficulty\": \"medium\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"MED-003\",\n", + " \"question\": \"What is the difference between Type 1 and Type 2 diabetes?\",\n", + " \"expected_answer\": \"Type 1 diabetes is an autoimmune condition where the body destroys insulin-producing cells. Type 2 is characterized by insulin resistance, often linked to lifestyle factors.\",\n", + " \"relevant_context\": \"Type 1 diabetes is an autoimmune disease where the immune system destroys pancreatic beta cells, eliminating insulin production. It typically appears in childhood/adolescence and requires lifelong insulin therapy. Type 2 diabetes involves insulin resistance \u2014 cells don't respond effectively to insulin \u2014 often combined with gradually declining insulin production. It accounts for ~90% of diabetes cases and is associated with obesity, sedentary lifestyle, and genetic factors.\",\n", + " \"domain\": \"medicine\",\n", + " \"difficulty\": \"medium\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"MED-004\",\n", + " \"question\": \"Can aspirin cure cancer?\",\n", + " \"expected_answer\": \"Aspirin cannot cure cancer. Some studies suggest it may reduce risk of certain cancers (particularly colorectal), but it is not a cancer treatment.\",\n", + " \"relevant_context\": \"Aspirin (acetylsalicylic acid) is a nonsteroidal anti-inflammatory drug (NSAID). Research has shown associations between regular low-dose aspirin use and reduced risk of colorectal cancer (approximately 20-30% reduction in some studies). However, aspirin is NOT a cancer treatment or cure. Its anti-cancer mechanisms likely involve COX-2 inhibition reducing inflammation. Current medical guidance does not recommend aspirin solely for cancer prevention due to bleeding risks.\",\n", + " \"domain\": \"medicine\",\n", + " \"difficulty\": \"hard\",\n", + " \"expected_hallucination_type\": \"factual\",\n", + " },\n", + " {\n", + " \"id\": \"MED-005\",\n", + " \"question\": \"What is the normal resting heart rate for adults?\",\n", + " \"expected_answer\": \"The normal resting heart rate for adults is between 60 and 100 beats per minute.\",\n", + " \"relevant_context\": \"A normal resting heart rate for adults ranges from 60 to 100 beats per minute (bpm). Well-trained athletes may have resting rates as low as 40-60 bpm due to more efficient cardiac function. Factors affecting heart rate include age, fitness level, body temperature, medications, stress, and body position. A consistently high resting heart rate (>100 bpm) is called tachycardia.\",\n", + " \"domain\": \"medicine\",\n", + " \"difficulty\": \"easy\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"MED-006\",\n", + " \"question\": \"How does antibiotic resistance develop?\",\n", + " \"expected_answer\": \"Antibiotic resistance develops through natural selection \u2014 bacteria with random mutations that confer resistance survive antibiotic exposure and reproduce, spreading resistant genes.\",\n", + " \"relevant_context\": \"Antibiotic resistance develops through evolutionary pressure. When bacteria are exposed to antibiotics, those with random genetic mutations conferring resistance survive and reproduce (natural selection). Resistance genes can spread through: vertical transfer (parent to offspring), horizontal gene transfer (conjugation, transformation, transduction between bacteria). Overuse and misuse of antibiotics accelerates this process. WHO considers antibiotic resistance one of the biggest threats to global health.\",\n", + " \"domain\": \"medicine\",\n", + " \"difficulty\": \"medium\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"MED-007\",\n", + " \"question\": \"What is the exact cure rate for stage 4 pancreatic cancer?\",\n", + " \"expected_answer\": \"Stage 4 pancreatic cancer has an extremely poor prognosis with a 5-year survival rate of approximately 1-3%. There is no guaranteed cure.\",\n", + " \"relevant_context\": \"Stage 4 (metastatic) pancreatic cancer has one of the lowest survival rates among cancers. The 5-year survival rate is approximately 1-3%. Median survival after diagnosis is 3-6 months with treatment. Treatments (chemotherapy combinations like FOLFIRINOX, gemcitabine+nab-paclitaxel) can extend survival but rarely achieve complete remission. Clinical trials continue to explore immunotherapy and targeted therapy approaches.\",\n", + " \"domain\": \"medicine\",\n", + " \"difficulty\": \"hard\",\n", + " \"expected_hallucination_type\": \"factual\",\n", + " },\n", + " {\n", + " \"id\": \"MED-008\",\n", + " \"question\": \"What causes Alzheimer's disease?\",\n", + " \"expected_answer\": \"The exact cause of Alzheimer's is not fully understood, but it involves accumulation of amyloid-beta plaques and tau tangles in the brain, along with genetic and environmental factors.\",\n", + " \"relevant_context\": \"Alzheimer's disease is a progressive neurodegenerative disorder. Its exact cause remains unclear, but key pathological features include: accumulation of amyloid-beta protein plaques between neurons, formation of neurofibrillary tangles of tau protein inside neurons, chronic neuroinflammation, and synaptic loss. Risk factors include age, genetics (APOE4 allele), cardiovascular health, and possibly environmental factors. The amyloid hypothesis is debated.\",\n", + " \"domain\": \"medicine\",\n", + " \"difficulty\": \"hard\",\n", + " \"expected_hallucination_type\": \"reasoning\",\n", + " },\n", + " {\n", + " \"id\": \"MED-009\",\n", + " \"question\": \"How long does it take for a broken bone to heal?\",\n", + " \"expected_answer\": \"Most broken bones take 6-8 weeks to heal substantially, though complete healing can take months depending on the bone and severity.\",\n", + " \"relevant_context\": \"Bone fracture healing time varies by location, severity, and patient factors. General timelines: finger/toe bones: 3-5 weeks; wrist/arm: 6-8 weeks; leg bones: 8-12 weeks; femur: 12-16 weeks. Healing occurs in stages: inflammation (days 1-5), soft callus formation (weeks 1-3), hard callus (weeks 3-8), and remodeling (months to years). Factors affecting healing include age, nutrition, blood supply, and whether the fracture is displaced.\",\n", + " \"domain\": \"medicine\",\n", + " \"difficulty\": \"easy\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"MED-010\",\n", + " \"question\": \"Is there a proven link between vaccines and autism?\",\n", + " \"expected_answer\": \"No. Extensive scientific research involving millions of children has found no link between vaccines and autism. The original 1998 study was fraudulent and retracted.\",\n", + " \"relevant_context\": \"There is no scientific evidence linking vaccines to autism. The claim originated from a 1998 study by Andrew Wakefield published in The Lancet, which was later found to be fraudulent \u2014 the paper was retracted in 2010, and Wakefield lost his medical license. Subsequent studies involving millions of children (including a 2019 Danish study of 657,461 children) have consistently found no association between vaccination and autism spectrum disorder.\",\n", + " \"domain\": \"medicine\",\n", + " \"difficulty\": \"medium\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " # === GEOGRAPHY DOMAIN (10 cases) ===\n", + " {\n", + " \"id\": \"GEO-001\",\n", + " \"question\": \"What is the longest river in the world?\",\n", + " \"expected_answer\": \"The Nile River is traditionally considered the longest river at approximately 6,650 km, though some measurements suggest the Amazon may be longer.\",\n", + " \"relevant_context\": \"The title of world's longest river is disputed between the Nile and the Amazon. The Nile, flowing through northeastern Africa, is traditionally measured at approximately 6,650 km (4,130 miles). Recent measurements of the Amazon, including its most distant source in Peru, suggest it may be 6,992 km (4,345 miles). The difference depends on how river length is measured (source definition, channel measurement methods).\",\n", + " \"domain\": \"geography\",\n", + " \"difficulty\": \"medium\",\n", + " \"expected_hallucination_type\": \"contextual\",\n", + " },\n", + " {\n", + " \"id\": \"GEO-002\",\n", + " \"question\": \"What is the capital of Australia?\",\n", + " \"expected_answer\": \"The capital of Australia is Canberra, not Sydney or Melbourne.\",\n", + " \"relevant_context\": \"The capital of Australia is Canberra, located in the Australian Capital Territory (ACT). It became the capital in 1913 as a compromise between rivals Sydney and Melbourne. Canberra was purpose-built as the capital city, designed by architects Walter Burley Griffin and Marion Mahony Griffin. Despite being the capital, it has a much smaller population (~450,000) than Sydney (~5.3 million) or Melbourne (~5.1 million).\",\n", + " \"domain\": \"geography\",\n", + " \"difficulty\": \"easy\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"GEO-003\",\n", + " \"question\": \"How deep is the Mariana Trench?\",\n", + " \"expected_answer\": \"The Mariana Trench reaches approximately 10,994 meters (36,070 feet) at its deepest point, Challenger Deep.\",\n", + " \"relevant_context\": \"The Mariana Trench in the western Pacific Ocean is the deepest known location on Earth. Its deepest point, Challenger Deep, has been measured at approximately 10,994 meters (36,070 feet) below sea level, with measurements varying between surveys (\u00b140 meters). It was first sounded in 1875 by HMS Challenger. Only four crewed descents have reached the bottom: Trieste (1960), Deepsea Challenger (2012), Limiting Factor (2019, multiple dives).\",\n", + " \"domain\": \"geography\",\n", + " \"difficulty\": \"easy\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"GEO-004\",\n", + " \"question\": \"Which countries does the Amazon River flow through?\",\n", + " \"expected_answer\": \"The Amazon River flows through Peru, Colombia, and Brazil, with tributaries in several other South American countries.\",\n", + " \"relevant_context\": \"The Amazon River's main channel flows through three countries: it originates in Peru (Andes Mountains), briefly passes through Colombia, and flows across Brazil to the Atlantic Ocean. Its vast drainage basin (7 million km\u00b2) also covers parts of Bolivia, Venezuela, Ecuador, Guyana, Suriname, and French Guiana. The river discharges approximately 209,000 cubic meters per second \u2014 more than the next seven largest rivers combined.\",\n", + " \"domain\": \"geography\",\n", + " \"difficulty\": \"medium\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"GEO-005\",\n", + " \"question\": \"What is the smallest country in the world?\",\n", + " \"expected_answer\": \"Vatican City is the smallest country in the world by both area (0.44 km\u00b2) and population (~800).\",\n", + " \"relevant_context\": \"Vatican City (officially Vatican City State) is the smallest internationally recognized independent state by both area and population. Located within Rome, Italy, it covers approximately 0.44 km\u00b2 (0.17 sq mi) and has a population of about 800. It became independent from Italy through the 1929 Lateran Treaty. The second smallest country is Monaco (2.02 km\u00b2).\",\n", + " \"domain\": \"geography\",\n", + " \"difficulty\": \"easy\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"GEO-006\",\n", + " \"question\": \"Why is the Dead Sea called 'dead'?\",\n", + " \"expected_answer\": \"The Dead Sea is called 'dead' because its extreme salinity (about 34%) prevents most organisms from living in it.\",\n", + " \"relevant_context\": \"The Dead Sea, bordered by Jordan, Israel, and the West Bank, is called 'dead' because its extreme salinity (~34.2%, nearly 10 times saltier than the ocean) prevents macroscopic aquatic organisms from thriving. However, it's not truly lifeless \u2014 certain halophilic (salt-loving) bacteria and archaea survive there. Its surface is about 430 meters below sea level, making it Earth's lowest land elevation. The sea has been shrinking due to water diversion from the Jordan River.\",\n", + " \"domain\": \"geography\",\n", + " \"difficulty\": \"medium\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"GEO-007\",\n", + " \"question\": \"How many countries are there in Africa?\",\n", + " \"expected_answer\": \"There are 54 recognized sovereign countries in Africa.\",\n", + " \"relevant_context\": \"Africa contains 54 internationally recognized sovereign states (as recognized by the United Nations and the African Union). The most recent addition was South Sudan, which gained independence from Sudan in 2011. The largest country by area is Algeria; the most populous is Nigeria (~220 million). Western Sahara's status remains disputed. Africa is the second-largest and second-most populous continent.\",\n", + " \"domain\": \"geography\",\n", + " \"difficulty\": \"easy\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"GEO-008\",\n", + " \"question\": \"What causes earthquakes?\",\n", + " \"expected_answer\": \"Earthquakes are caused by sudden release of energy in the Earth's crust, usually due to movement along tectonic plate boundaries or fault lines.\",\n", + " \"relevant_context\": \"Earthquakes occur when energy stored in Earth's crust is suddenly released, usually due to movement along geological faults. Tectonic plates constantly move (plate tectonics), and stress accumulates at plate boundaries. When friction is overcome, plates slip suddenly, releasing seismic waves. Most earthquakes occur at plate boundaries: transform (San Andreas), convergent (subduction zones), and divergent boundaries. Intraplate earthquakes can also occur along ancient fault lines.\",\n", + " \"domain\": \"geography\",\n", + " \"difficulty\": \"medium\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"GEO-009\",\n", + " \"question\": \"What is the exact current population of Tokyo?\",\n", + " \"expected_answer\": \"Tokyo's population is approximately 14 million in the city proper, or about 37 million in the greater metropolitan area, making it the world's most populous metropolitan area.\",\n", + " \"relevant_context\": \"Tokyo's population depends on the boundary definition used. Tokyo Metropolis (administrative boundary) has approximately 14 million residents. The Greater Tokyo Area (Tokyo plus surrounding prefectures of Kanagawa, Saitama, and Chiba) has approximately 37-38 million people, making it the world's largest metropolitan area by population. Population figures change continuously and exact numbers depend on the census year and counting methodology.\",\n", + " \"domain\": \"geography\",\n", + " \"difficulty\": \"hard\",\n", + " \"expected_hallucination_type\": \"contextual\",\n", + " },\n", + " {\n", + " \"id\": \"GEO-010\",\n", + " \"question\": \"Is Greenland a continent or an island?\",\n", + " \"expected_answer\": \"Greenland is classified as an island (the world's largest) rather than a continent, primarily due to convention and its geological relationship to the North American plate.\",\n", + " \"relevant_context\": \"Greenland is classified as the world's largest island (approximately 2.166 million km\u00b2) rather than a continent. The distinction is somewhat arbitrary \u2014 Australia (7.692 million km\u00b2) is the smallest continent. The classification depends on tectonic plate structure, geological history, and convention. Greenland sits on the North American tectonic plate and is geologically part of the Canadian Shield. Despite its size, it has only about 56,000 inhabitants, mostly along the coast.\",\n", + " \"domain\": \"geography\",\n", + " \"difficulty\": \"hard\",\n", + " \"expected_hallucination_type\": \"reasoning\",\n", + " },\n", + " # === ADDITIONAL CASES FOR DIVERSITY (5 cases) ===\n", + " {\n", + " \"id\": \"MIX-001\",\n", + " \"question\": \"What is the meaning of life according to Douglas Adams?\",\n", + " \"expected_answer\": \"In Douglas Adams' 'The Hitchhiker's Guide to the Galaxy', the answer to the ultimate question of life, the universe, and everything is 42.\",\n", + " \"relevant_context\": \"In Douglas Adams' science fiction series 'The Hitchhiker's Guide to the Galaxy' (1979), a supercomputer named Deep Thought computes for 7.5 million years and determines that the Answer to the Ultimate Question of Life, the Universe, and Everything is simply the number 42. The joke is that the actual Question was never properly defined.\",\n", + " \"domain\": \"culture\",\n", + " \"difficulty\": \"easy\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"MIX-002\",\n", + " \"question\": \"Can humans survive without sleep?\",\n", + " \"expected_answer\": \"No. Prolonged sleep deprivation causes severe cognitive impairment and can eventually be fatal. The longest documented case of intentional wakefulness was about 11 days.\",\n", + " \"relevant_context\": \"Humans cannot survive indefinitely without sleep. Sleep deprivation causes progressive cognitive decline, hallucinations, immune suppression, and potentially death. The longest scientifically documented period without sleep is approximately 264 hours (11 days) by Randy Gardner in 1964, though he experienced severe cognitive and perceptual disturbances. Fatal familial insomnia (FFI) demonstrates that complete inability to sleep is ultimately lethal.\",\n", + " \"domain\": \"biology\",\n", + " \"difficulty\": \"medium\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"MIX-003\",\n", + " \"question\": \"What is the current world record for the 100m sprint?\",\n", + " \"expected_answer\": \"The world record for the 100m sprint is 9.58 seconds, set by Usain Bolt at the 2009 World Championships in Berlin.\",\n", + " \"relevant_context\": \"The men's 100 meters world record is 9.58 seconds, set by Jamaican sprinter Usain Bolt on August 16, 2009, at the World Championships in Berlin, Germany. He broke his own previous record of 9.69 seconds from the 2008 Beijing Olympics. The women's record is 10.49 seconds by Florence Griffith-Joyner (1988). Bolt also holds the 200m record (19.19s) set at the same 2009 championships.\",\n", + " \"domain\": \"sports\",\n", + " \"difficulty\": \"easy\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"MIX-004\",\n", + " \"question\": \"How do noise-cancelling headphones work?\",\n", + " \"expected_answer\": \"Noise-cancelling headphones use microphones to detect ambient sound and generate an 'anti-noise' signal (inverted phase) that destructively interferes with incoming noise.\",\n", + " \"relevant_context\": \"Active noise-cancelling (ANC) headphones work through destructive interference. External microphones capture ambient sound waves. A processor analyzes these waves and generates an 'anti-noise' signal \u2014 a sound wave with the same amplitude but inverted phase (180\u00b0 out of phase). When the anti-noise meets the original noise, they cancel each other out (destructive interference). This works best for consistent, low-frequency sounds (airplane engines, HVAC hum) and less effectively for irregular, high-frequency sounds (speech).\",\n", + " \"domain\": \"technology\",\n", + " \"difficulty\": \"medium\",\n", + " \"expected_hallucination_type\": \"faithful\",\n", + " },\n", + " {\n", + " \"id\": \"MIX-005\",\n", + " \"question\": \"What would happen if the Earth stopped rotating?\",\n", + " \"expected_answer\": \"If Earth suddenly stopped rotating, catastrophic effects would include extreme winds, massive tsunamis, redistribution of oceans toward the poles, and loss of the magnetic field.\",\n", + " \"relevant_context\": \"If Earth suddenly stopped rotating, the consequences would be catastrophic: (1) Everything not attached to bedrock would continue moving at rotational speed (~1,670 km/h at equator), causing extreme winds and displacement; (2) Oceans would redistribute toward the poles due to loss of centrifugal force, creating a mega-continent at the equator; (3) Day/night cycles would become ~6 months each; (4) The magnetic field (generated by the rotating liquid core) would likely weaken significantly; (5) Climate would become extreme with one side perpetually heated.\",\n", + " \"domain\": \"science\",\n", + " \"difficulty\": \"hard\",\n", + " \"expected_hallucination_type\": \"reasoning\",\n", + " },\n", + " ]\n", + "\n", + " # Validate dataset\n", + " assert len(dataset) >= 50, f\"Dataset has only {len(dataset)} cases (need >= 50)\"\n", + " ids = [d[\"id\"] for d in dataset]\n", + " assert len(ids) == len(set(ids)), \"Duplicate IDs found\"\n", + "\n", + " return dataset\n", + "\n", + "\n", + "# Create and summarize dataset\n", + "eval_dataset = create_synthetic_dataset()\n", + "print(f\"\u2713 Created synthetic dataset with {len(eval_dataset)} test cases\")\n", + "print(f\" Domains: {Counter(d['domain'] for d in eval_dataset)}\")\n", + "print(f\" Difficulties: {Counter(d['difficulty'] for d in eval_dataset)}\")\n", + "print(f\" Hallucination types: {Counter(d['expected_hallucination_type'] for d in eval_dataset)}\")\n", + "print()\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 3: Retrieval Strategies (Controlled Experimentation)\n", + "\n", + "Implement and compare 3 retrieval strategies:\n", + "1. **Hybrid:** BM25 + FAISS with Reciprocal Rank Fusion (RRF)\n", + "2. **Dense-only:** FAISS semantic search\n", + "3. **Two-stage:** FAISS \u2192 top-k \u2192 rerank with combined scoring\n", + "\n", + "Each returns top-k documents with similarity scores." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# \u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n", + "# \u2551 SECTION 3: RETRIEVAL STRATEGIES (CONTROLLED EXPERIMENTATION) \u2551\n", + "# \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n", + "\n", + "print(\"=\" * 70)\n", + "print(\"SECTION 3: Building Retrieval Systems\")\n", + "print(\"=\" * 70)\n", + "\n", + "\n", + "class RetrievalSystem:\n", + " \"\"\"\n", + " Manages the knowledge base and provides multiple retrieval strategies.\n", + " \"\"\"\n", + "\n", + " def __init__(self, embedding_model_name: str = \"sentence-transformers/all-MiniLM-L6-v2\"):\n", + " print(f\" Loading embedding model: {embedding_model_name}\")\n", + " self.embedding_model = SentenceTransformer(embedding_model_name)\n", + " self.embedding_dim = 384 # all-MiniLM-L6-v2 dimension\n", + "\n", + " # Storage\n", + " self.documents: List[str] = []\n", + " self.doc_metadata: List[Dict] = []\n", + "\n", + " # Indices\n", + " self.faiss_index: Optional[faiss.IndexFlatIP] = None\n", + " self.bm25: Optional[BM25Okapi] = None\n", + " self.embeddings: Optional[np.ndarray] = None\n", + "\n", + " def build_index(self, documents: List[str], metadata: List[Dict] = None):\n", + " \"\"\"Build both FAISS and BM25 indices from documents.\"\"\"\n", + " self.documents = documents\n", + " self.doc_metadata = metadata or [{\"idx\": i} for i in range(len(documents))]\n", + "\n", + " # Build FAISS index (dense)\n", + " print(\" Building FAISS index...\")\n", + " self.embeddings = self.embedding_model.encode(\n", + " documents,\n", + " batch_size=32,\n", + " normalize_embeddings=True,\n", + " convert_to_numpy=True,\n", + " show_progress_bar=False,\n", + " )\n", + " self.faiss_index = faiss.IndexFlatIP(self.embedding_dim)\n", + " self.faiss_index.add(self.embeddings.astype(\"float32\"))\n", + "\n", + " # Build BM25 index (sparse)\n", + " print(\" Building BM25 index...\")\n", + " tokenized_corpus = [doc.lower().split() for doc in documents]\n", + " self.bm25 = BM25Okapi(tokenized_corpus)\n", + "\n", + " print(f\" \u2713 Indexed {len(documents)} documents\")\n", + "\n", + " def retrieve_hybrid(self, query: str, k: int = 5, rrf_k: int = 60) -> List[Dict]:\n", + " \"\"\"\n", + " Strategy 1: Hybrid BM25 + FAISS with Reciprocal Rank Fusion.\n", + " Combines sparse (lexical) and dense (semantic) retrieval.\n", + " \"\"\"\n", + " # Dense retrieval\n", + " q_emb = self.embedding_model.encode(\n", + " [query], normalize_embeddings=True, convert_to_numpy=True\n", + " ).astype(\"float32\")\n", + " dense_scores, dense_indices = self.faiss_index.search(q_emb, k)\n", + " dense_ranking = dense_indices[0].tolist()\n", + " dense_score_map = {idx: float(score) for idx, score in zip(dense_indices[0], dense_scores[0])}\n", + "\n", + " # Sparse retrieval\n", + " bm25_scores = self.bm25.get_scores(query.lower().split())\n", + " sparse_ranking = np.argsort(bm25_scores)[::-1][:k].tolist()\n", + "\n", + " # Reciprocal Rank Fusion\n", + " fused_scores = {}\n", + " for rank, idx in enumerate(dense_ranking):\n", + " if idx >= 0: # FAISS can return -1 for empty results\n", + " fused_scores[idx] = fused_scores.get(idx, 0) + 1.0 / (rrf_k + rank + 1)\n", + " for rank, idx in enumerate(sparse_ranking):\n", + " fused_scores[idx] = fused_scores.get(idx, 0) + 1.0 / (rrf_k + rank + 1)\n", + "\n", + " # Sort by fused score\n", + " ranked = sorted(fused_scores.items(), key=lambda x: x[1], reverse=True)[:k]\n", + "\n", + " results = []\n", + " for idx, fused_score in ranked:\n", + " results.append({\n", + " \"document\": self.documents[idx],\n", + " \"index\": idx,\n", + " \"score\": fused_score,\n", + " \"dense_score\": dense_score_map.get(idx, 0.0),\n", + " \"strategy\": \"hybrid\",\n", + " })\n", + " return results\n", + "\n", + " def retrieve_dense(self, query: str, k: int = 5) -> List[Dict]:\n", + " \"\"\"\n", + " Strategy 2: Dense-only retrieval using FAISS (semantic search).\n", + " \"\"\"\n", + " q_emb = self.embedding_model.encode(\n", + " [query], normalize_embeddings=True, convert_to_numpy=True\n", + " ).astype(\"float32\")\n", + " scores, indices = self.faiss_index.search(q_emb, k)\n", + "\n", + " results = []\n", + " for idx, score in zip(indices[0], scores[0]):\n", + " if idx >= 0:\n", + " results.append({\n", + " \"document\": self.documents[idx],\n", + " \"index\": int(idx),\n", + " \"score\": float(score),\n", + " \"dense_score\": float(score),\n", + " \"strategy\": \"dense\",\n", + " })\n", + " return results\n", + "\n", + " def retrieve_two_stage(self, query: str, k: int = 5, first_stage_k: int = 20) -> List[Dict]:\n", + " \"\"\"\n", + " Strategy 3: Two-stage retrieval.\n", + " Stage 1: FAISS retrieves top-k_first candidates.\n", + " Stage 2: Re-rank candidates using cross-encoder-style similarity scoring.\n", + " \"\"\"\n", + " # Stage 1: Broad FAISS retrieval\n", + " q_emb = self.embedding_model.encode(\n", + " [query], normalize_embeddings=True, convert_to_numpy=True\n", + " ).astype(\"float32\")\n", + " scores, indices = self.faiss_index.search(q_emb, first_stage_k)\n", + "\n", + " candidates = []\n", + " for idx, score in zip(indices[0], scores[0]):\n", + " if idx >= 0:\n", + " candidates.append((int(idx), float(score)))\n", + "\n", + " # Stage 2: Re-rank using token overlap + embedding similarity (lightweight cross-encoder proxy)\n", + " query_tokens = set(query.lower().split())\n", + " reranked = []\n", + " for idx, dense_score in candidates:\n", + " doc = self.documents[idx]\n", + " doc_tokens = set(doc.lower().split())\n", + "\n", + " # Token overlap (Jaccard-like)\n", + " overlap = len(query_tokens & doc_tokens) / max(len(query_tokens | doc_tokens), 1)\n", + "\n", + " # Combined score: weighted sum of dense similarity and token overlap\n", + " combined_score = 0.7 * dense_score + 0.3 * overlap\n", + " reranked.append((idx, combined_score, dense_score))\n", + "\n", + " # Sort by combined score\n", + " reranked.sort(key=lambda x: x[1], reverse=True)\n", + " results = []\n", + " for idx, combined_score, dense_score in reranked[:k]:\n", + " results.append({\n", + " \"document\": self.documents[idx],\n", + " \"index\": idx,\n", + " \"score\": combined_score,\n", + " \"dense_score\": dense_score,\n", + " \"strategy\": \"two_stage\",\n", + " })\n", + " return results\n", + "\n", + " def add_document(self, text: str, metadata: Dict = None):\n", + " \"\"\"\n", + " Add a new document to the knowledge base (dynamic update).\n", + " Updates both FAISS and BM25 indices.\n", + " \"\"\"\n", + " self.documents.append(text)\n", + " self.doc_metadata.append(metadata or {\"idx\": len(self.documents) - 1})\n", + "\n", + " # Update FAISS\n", + " new_embedding = self.embedding_model.encode(\n", + " [text], normalize_embeddings=True, convert_to_numpy=True\n", + " ).astype(\"float32\")\n", + " self.faiss_index.add(new_embedding)\n", + " self.embeddings = np.vstack([self.embeddings, new_embedding])\n", + "\n", + " # Rebuild BM25 (BM25Okapi doesn't support incremental updates)\n", + " tokenized_corpus = [doc.lower().split() for doc in self.documents]\n", + " self.bm25 = BM25Okapi(tokenized_corpus)\n", + "\n", + "\n", + "# Build knowledge base from dataset contexts\n", + "print(\" Initializing retrieval system...\")\n", + "retrieval_system = RetrievalSystem()\n", + "\n", + "# Extract all contexts as our knowledge base\n", + "knowledge_base = [item[\"relevant_context\"] for item in eval_dataset]\n", + "kb_metadata = [{\"id\": item[\"id\"], \"domain\": item[\"domain\"]} for item in eval_dataset]\n", + "retrieval_system.build_index(knowledge_base, kb_metadata)\n", + "print()\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 4 & 5: RAG Pipeline + Models\n", + "\n", + "**Pipeline:** query \u2192 retrieval \u2192 generation \u2192 verification\n", + "\n", + "**Models:**\n", + "1. **Primary:** TinyLlama-1.1B-Chat (lightweight open-source, chat-capable)\n", + "2. **Secondary:** DistilGPT2 (very small, fast baseline)\n", + "3. **Fallback:** Rule-based extractive generator (always available)\n", + "\n", + "Rules: Answer ONLY from retrieved context. If insufficient context \u2192 \"I don't know\"" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# \u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n", + "# \u2551 SECTION 4 & 5: RAG PIPELINE + MODELS \u2551\n", + "# \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n", + "\n", + "print(\"=\" * 70)\n", + "print(\"SECTION 4 & 5: RAG Pipeline + Model Loading\")\n", + "print(\"=\" * 70)\n", + "\n", + "\n", + "class GenerationModel:\n", + " \"\"\"Wrapper for text generation models.\"\"\"\n", + "\n", + " def __init__(self, model_name: str, model_type: str = \"pipeline\"):\n", + " self.model_name = model_name\n", + " self.model_type = model_type\n", + " self.pipe = None\n", + " self.loaded = False\n", + "\n", + " def load(self):\n", + " \"\"\"Load the model. Handles failures gracefully.\"\"\"\n", + " try:\n", + " print(f\" Loading model: {self.model_name}...\")\n", + " device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", + " dtype = torch.bfloat16 if torch.cuda.is_available() else torch.float32\n", + "\n", + " self.pipe = pipeline(\n", + " \"text-generation\",\n", + " model=self.model_name,\n", + " torch_dtype=dtype,\n", + " device_map=\"auto\" if torch.cuda.is_available() else None,\n", + " device=None if torch.cuda.is_available() else device,\n", + " )\n", + " self.loaded = True\n", + " print(f\" \u2713 Model loaded: {self.model_name}\")\n", + " except Exception as e:\n", + " print(f\" \u2717 Failed to load {self.model_name}: {e}\")\n", + " self.loaded = False\n", + "\n", + " def generate(self, prompt: str, max_new_tokens: int = 256) -> str:\n", + " \"\"\"Generate text from prompt.\"\"\"\n", + " if not self.loaded:\n", + " return \"[MODEL NOT LOADED]\"\n", + " try:\n", + " outputs = self.pipe(\n", + " prompt,\n", + " max_new_tokens=max_new_tokens,\n", + " do_sample=False,\n", + " repetition_penalty=1.1,\n", + " return_full_text=False,\n", + " pad_token_id=self.pipe.tokenizer.eos_token_id,\n", + " )\n", + " return outputs[0][\"generated_text\"].strip()\n", + " except Exception as e:\n", + " return f\"[GENERATION ERROR: {e}]\"\n", + "\n", + "\n", + "class SimpleGeneratorFallback:\n", + " \"\"\"\n", + " Fallback generator that uses extractive summarization from context.\n", + " Does not require GPU \u2014 purely rule-based.\n", + " \"\"\"\n", + "\n", + " def __init__(self):\n", + " self.model_name = \"extractive-fallback\"\n", + " self.loaded = True\n", + "\n", + " def load(self):\n", + " pass\n", + "\n", + " def generate(self, prompt: str, max_new_tokens: int = 256) -> str:\n", + " \"\"\"Extract most relevant sentence from context as answer.\"\"\"\n", + " # Parse context from prompt\n", + " context_start = prompt.find(\"Context:\")\n", + " question_start = prompt.find(\"Question:\")\n", + "\n", + " if context_start == -1 or question_start == -1:\n", + " return \"I don't know based on the provided context.\"\n", + "\n", + " context = prompt[context_start + 8:question_start].strip()\n", + " question = prompt[question_start + 9:].strip()\n", + "\n", + " # Simple extractive approach: find sentence with most keyword overlap\n", + " sentences = [s.strip() for s in context.replace(\"\\n\", \" \").split(\".\") if len(s.strip()) > 10]\n", + " if not sentences:\n", + " return \"I don't know based on the provided context.\"\n", + "\n", + " question_words = set(question.lower().split())\n", + " best_sentence = \"\"\n", + " best_score = 0\n", + "\n", + " for sent in sentences:\n", + " sent_words = set(sent.lower().split())\n", + " overlap = len(question_words & sent_words)\n", + " if overlap > best_score:\n", + " best_score = overlap\n", + " best_sentence = sent\n", + "\n", + " if best_score == 0:\n", + " return \"I don't know based on the provided context.\"\n", + "\n", + " return best_sentence.strip() + \".\"\n", + "\n", + "\n", + "# Initialize models\n", + "print()\n", + "\n", + "# Model 1: TinyLlama (lightweight, open-source, chat-capable)\n", + "model_primary = GenerationModel(\"TinyLlama/TinyLlama-1.1B-Chat-v1.0\", model_type=\"pipeline\")\n", + "model_primary.load()\n", + "\n", + "# Model 2: DistilGPT2 (very small, fast, good for comparison)\n", + "model_secondary = GenerationModel(\"distilgpt2\", model_type=\"pipeline\")\n", + "model_secondary.load()\n", + "\n", + "# Model 3: Extractive fallback (rule-based, always available)\n", + "model_fallback = SimpleGeneratorFallback()\n", + "\n", + "# Collect available models\n", + "available_models = []\n", + "if model_primary.loaded:\n", + " available_models.append((\"TinyLlama-1.1B-Chat\", model_primary))\n", + "if model_secondary.loaded:\n", + " available_models.append((\"DistilGPT2\", model_secondary))\n", + "available_models.append((\"Extractive-Fallback\", model_fallback))\n", + "\n", + "print(f\"\\n \u2713 {len(available_models)} models available: {[m[0] for m in available_models]}\")\n", + "print()\n", + "\n", + "\n", + "def build_rag_prompt(question: str, contexts: List[str]) -> str:\n", + " \"\"\"Build a RAG prompt with retrieved context.\"\"\"\n", + " context_text = \"\\n\\n\".join(contexts)\n", + " prompt = f\"\"\"Answer the question using ONLY the context provided below. If the context does not contain enough information to answer the question, respond with \"I don't know based on the provided context.\"\n", + "\n", + "Context:\n", + "{context_text}\n", + "\n", + "Question: {question}\n", + "\n", + "Answer:\"\"\"\n", + " return prompt\n", + "\n", + "\n", + "def build_chat_prompt(question: str, contexts: List[str], tokenizer=None) -> str:\n", + " \"\"\"Build a chat-formatted prompt for chat models.\"\"\"\n", + " context_text = \"\\n\\n\".join(contexts)\n", + " messages = [\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": \"You are a helpful assistant. Answer questions using ONLY the provided context. If the context is insufficient, say 'I don't know based on the provided context.' Be concise and factual.\",\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": f\"Context:\\n{context_text}\\n\\nQuestion: {question}\",\n", + " },\n", + " ]\n", + " if tokenizer and hasattr(tokenizer, \"apply_chat_template\"):\n", + " try:\n", + " return tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)\n", + " except Exception:\n", + " pass\n", + " return build_rag_prompt(question, contexts)\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 6: Agent System (Lightweight)\n", + "\n", + "4 agents implemented as simple classes (no external frameworks):\n", + "\n", + "| Agent | Role |\n", + "|-------|------|\n", + "| **PlannerAgent** | Rule-based step planning (retrieve \u2192 generate \u2192 verify) |\n", + "| **ExecutorAgent** | Executes retrieval and generation, calls tools |\n", + "| **CriticAgent** | Evaluates grounding, assigns hallucination type + confidence |\n", + "| **LoggingAgent** | Records all system activity with structured prefixes |" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# \u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n", + "# \u2551 SECTION 6: AGENT SYSTEM (LIGHTWEIGHT) \u2551\n", + "# \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n", + "\n", + "print(\"=\" * 70)\n", + "print(\"SECTION 6: Agent System\")\n", + "print(\"=\" * 70)\n", + "\n", + "\n", + "class LoggingAgent:\n", + " \"\"\"\n", + " Records all system activity for analysis.\n", + " Provides structured logging with prefixed messages.\n", + " \"\"\"\n", + "\n", + " def __init__(self):\n", + " self.logs: List[Dict] = []\n", + "\n", + " def log(self, agent: str, action: str, details: Dict = None):\n", + " \"\"\"Log an agent action with timestamp.\"\"\"\n", + " entry = {\n", + " \"timestamp\": time.time(),\n", + " \"agent\": agent,\n", + " \"action\": action,\n", + " \"details\": details or {},\n", + " }\n", + " self.logs.append(entry)\n", + " prefix = f\"[{agent.upper()}]\"\n", + " print(f\" {prefix:12s} {action}\")\n", + "\n", + " def get_logs(self) -> List[Dict]:\n", + " \"\"\"Return all logs.\"\"\"\n", + " return self.logs\n", + "\n", + " def get_summary(self) -> Dict:\n", + " \"\"\"Return summary statistics.\"\"\"\n", + " return {\n", + " \"total_entries\": len(self.logs),\n", + " \"by_agent\": Counter(log[\"agent\"] for log in self.logs),\n", + " \"by_action\": Counter(log[\"action\"] for log in self.logs),\n", + " }\n", + "\n", + " def clear(self):\n", + " \"\"\"Clear all logs.\"\"\"\n", + " self.logs = []\n", + "\n", + "\n", + "class PlannerAgent:\n", + " \"\"\"\n", + " Rule-based planner that decides the execution steps.\n", + " Analyzes the query and determines the optimal workflow.\n", + " \"\"\"\n", + "\n", + " def __init__(self, logger: LoggingAgent):\n", + " self.logger = logger\n", + "\n", + " def plan(self, query: str, retrieval_strategy: str = \"hybrid\") -> List[Dict]:\n", + " \"\"\"\n", + " Create an execution plan based on the query.\n", + " Returns ordered list of steps.\n", + " \"\"\"\n", + " self.logger.log(\"PLANNER\", \"Planning started\", {\"query\": query[:80]})\n", + "\n", + " steps = [\n", + " {\"step\": 1, \"action\": \"retrieve\", \"strategy\": retrieval_strategy, \"k\": 5},\n", + " {\"step\": 2, \"action\": \"generate\", \"model\": \"primary\"},\n", + " {\"step\": 3, \"action\": \"verify\", \"checks\": [\"grounding\", \"confidence\"]},\n", + " ]\n", + "\n", + " # Adaptive planning based on query complexity\n", + " query_words = query.split()\n", + " if len(query_words) > 15 or \"?\" in query and query.count(\"?\") > 1:\n", + " # Complex query: add extra retrieval\n", + " steps[0][\"k\"] = 7\n", + " self.logger.log(\"PLANNER\", \"Complex query detected \u2014 increased k to 7\")\n", + "\n", + " if any(word in query.lower() for word in [\"exact\", \"precise\", \"specific\", \"how many\"]):\n", + " # Precision-required query: add stricter verification\n", + " steps[2][\"checks\"].append(\"precision_check\")\n", + " self.logger.log(\"PLANNER\", \"Precision query \u2014 added strict verification\")\n", + "\n", + " self.logger.log(\"PLANNER\", f\"Plan created: {len(steps)} steps\", {\"steps\": steps})\n", + " return steps\n", + "\n", + "\n", + "class ExecutorAgent:\n", + " \"\"\"\n", + " Executes retrieval and generation steps from the plan.\n", + " Calls tools and manages the pipeline execution.\n", + " \"\"\"\n", + "\n", + " def __init__(self, retrieval_system: RetrievalSystem, models: List[tuple], logger: LoggingAgent):\n", + " self.retrieval_system = retrieval_system\n", + " self.models = {name: model for name, model in models}\n", + " self.logger = logger\n", + "\n", + " def execute(self, plan: List[Dict], query: str) -> Dict:\n", + " \"\"\"Execute the plan and return results.\"\"\"\n", + " result = {\"query\": query, \"steps_executed\": []}\n", + "\n", + " for step in plan:\n", + " action = step[\"action\"]\n", + " start_time = time.time()\n", + "\n", + " if action == \"retrieve\":\n", + " retrieved = self._execute_retrieval(query, step[\"strategy\"], step.get(\"k\", 5))\n", + " result[\"retrieved_docs\"] = retrieved\n", + " result[\"retrieval_strategy\"] = step[\"strategy\"]\n", + "\n", + " elif action == \"generate\":\n", + " contexts = [doc[\"document\"] for doc in result.get(\"retrieved_docs\", [])]\n", + " answer = self._execute_generation(query, contexts)\n", + " result[\"generated_answer\"] = answer\n", + "\n", + " elif action == \"verify\":\n", + " # Verification handled by CriticAgent\n", + " pass\n", + "\n", + " elapsed = time.time() - start_time\n", + " result[\"steps_executed\"].append({\"action\": action, \"latency\": elapsed})\n", + " self.logger.log(\"EXECUTOR\", f\"Step '{action}' completed\", {\"latency_ms\": round(elapsed * 1000, 1)})\n", + "\n", + " return result\n", + "\n", + " def _execute_retrieval(self, query: str, strategy: str, k: int) -> List[Dict]:\n", + " \"\"\"Execute retrieval using specified strategy.\"\"\"\n", + " self.logger.log(\"EXECUTOR\", f\"Retrieving with strategy='{strategy}', k={k}\")\n", + "\n", + " if strategy == \"hybrid\":\n", + " return self.retrieval_system.retrieve_hybrid(query, k=k)\n", + " elif strategy == \"dense\":\n", + " return self.retrieval_system.retrieve_dense(query, k=k)\n", + " elif strategy == \"two_stage\":\n", + " return self.retrieval_system.retrieve_two_stage(query, k=k)\n", + " else:\n", + " self.logger.log(\"EXECUTOR\", f\"Unknown strategy '{strategy}', falling back to hybrid\")\n", + " return self.retrieval_system.retrieve_hybrid(query, k=k)\n", + "\n", + " def _execute_generation(self, query: str, contexts: List[str]) -> str:\n", + " \"\"\"Generate answer using the best available model.\"\"\"\n", + " if not contexts:\n", + " return \"I don't know based on the provided context.\"\n", + "\n", + " # Use primary model with chat template if available\n", + " model_name, model = available_models[0]\n", + "\n", + " if hasattr(model, \"pipe\") and model.pipe and hasattr(model.pipe, \"tokenizer\"):\n", + " prompt = build_chat_prompt(query, contexts, model.pipe.tokenizer)\n", + " else:\n", + " prompt = build_rag_prompt(query, contexts)\n", + "\n", + " answer = model.generate(prompt, max_new_tokens=200)\n", + " self.logger.log(\"EXECUTOR\", f\"Generated answer using {model_name}\", {\"answer_length\": len(answer)})\n", + " return answer\n", + "\n", + "\n", + "class CriticAgent:\n", + " \"\"\"\n", + " Evaluates answer quality, grounding, and hallucination.\n", + " Assigns hallucination type and confidence score.\n", + " \"\"\"\n", + "\n", + " def __init__(self, logger: LoggingAgent):\n", + " self.logger = logger\n", + "\n", + " def evaluate(self, query: str, answer: str, contexts: List[str]) -> Dict:\n", + " \"\"\"\n", + " Evaluate the generated answer against retrieved context.\n", + " Returns grounding assessment, hallucination classification, and confidence.\n", + " \"\"\"\n", + " self.logger.log(\"CRITIC\", \"Evaluating answer quality\")\n", + "\n", + " # Compute context overlap\n", + " overlap_score = self._compute_context_overlap(answer, contexts)\n", + "\n", + " # Grounding check\n", + " grounding = self._check_grounding(answer, contexts)\n", + "\n", + " # Hallucination classification\n", + " hallucination_type = self._classify_hallucination(answer, contexts, overlap_score)\n", + "\n", + " # Confidence score (heuristic)\n", + " confidence = self._compute_confidence(overlap_score, grounding, answer)\n", + "\n", + " evaluation = {\n", + " \"context_overlap_score\": overlap_score,\n", + " \"grounding\": grounding,\n", + " \"hallucination_type\": hallucination_type,\n", + " \"confidence_score\": confidence,\n", + " }\n", + "\n", + " self.logger.log(\"CRITIC\", f\"Evaluation complete\", {\n", + " \"overlap\": round(overlap_score, 3),\n", + " \"confidence\": round(confidence, 3),\n", + " \"hallucination\": hallucination_type,\n", + " })\n", + "\n", + " return evaluation\n", + "\n", + " def _compute_context_overlap(self, answer: str, contexts: List[str]) -> float:\n", + " \"\"\"Compute token-level overlap between answer and contexts.\"\"\"\n", + " if not answer or not contexts:\n", + " return 0.0\n", + "\n", + " answer_tokens = set(answer.lower().split())\n", + " context_tokens = set()\n", + " for ctx in contexts:\n", + " context_tokens.update(ctx.lower().split())\n", + "\n", + " # Remove common stopwords for meaningful overlap\n", + " stopwords = {\"the\", \"a\", \"an\", \"is\", \"are\", \"was\", \"were\", \"in\", \"on\", \"at\", \"to\",\n", + " \"for\", \"of\", \"and\", \"or\", \"it\", \"its\", \"this\", \"that\", \"with\", \"by\",\n", + " \"from\", \"as\", \"be\", \"has\", \"had\", \"have\", \"not\", \"but\", \"if\", \"they\"}\n", + "\n", + " answer_meaningful = answer_tokens - stopwords\n", + " context_meaningful = context_tokens - stopwords\n", + "\n", + " if not answer_meaningful:\n", + " return 0.0\n", + "\n", + " overlap = len(answer_meaningful & context_meaningful) / len(answer_meaningful)\n", + " return overlap\n", + "\n", + " def _check_grounding(self, answer: str, contexts: List[str]) -> Dict:\n", + " \"\"\"Check if key claims in the answer are grounded in context.\"\"\"\n", + " if not answer or not contexts:\n", + " return {\"grounded\": False, \"coverage\": 0.0}\n", + "\n", + " combined_context = \" \".join(contexts).lower()\n", + " answer_sentences = [s.strip() for s in answer.split(\".\") if len(s.strip()) > 5]\n", + "\n", + " grounded_count = 0\n", + " for sent in answer_sentences:\n", + " # Check if key words from sentence appear in context\n", + " sent_words = set(sent.lower().split()) - {\"the\", \"a\", \"an\", \"is\", \"are\", \"was\", \"it\"}\n", + " if len(sent_words) == 0:\n", + " continue\n", + " context_words = set(combined_context.split())\n", + " match_ratio = len(sent_words & context_words) / len(sent_words)\n", + " if match_ratio > 0.5:\n", + " grounded_count += 1\n", + "\n", + " coverage = grounded_count / max(len(answer_sentences), 1)\n", + " return {\"grounded\": coverage > 0.6, \"coverage\": coverage}\n", + "\n", + " def _classify_hallucination(self, answer: str, contexts: List[str], overlap: float) -> str:\n", + " \"\"\"\n", + " Classify the type of potential hallucination.\n", + " Categories: factual, contextual, reasoning, faithful\n", + " \"\"\"\n", + " if overlap > 0.7:\n", + " return \"faithful\"\n", + " elif overlap > 0.5:\n", + " # Partially grounded \u2014 check if it's contextual or reasoning\n", + " if any(word in answer.lower() for word in [\"because\", \"therefore\", \"thus\", \"since\", \"implies\"]):\n", + " return \"reasoning\"\n", + " else:\n", + " return \"contextual\"\n", + " elif overlap > 0.3:\n", + " return \"contextual\"\n", + " else:\n", + " return \"factual\"\n", + "\n", + " def _compute_confidence(self, overlap: float, grounding: Dict, answer: str) -> float:\n", + " \"\"\"\n", + " Compute a heuristic confidence score [0, 1].\n", + " Higher = more likely to be correct and grounded.\n", + " \"\"\"\n", + " # Base confidence from overlap\n", + " conf = overlap * 0.5\n", + "\n", + " # Bonus for grounding\n", + " if grounding.get(\"grounded\", False):\n", + " conf += 0.3\n", + " else:\n", + " conf += grounding.get(\"coverage\", 0) * 0.2\n", + "\n", + " # Penalty for \"I don't know\" (but it's actually good behavior)\n", + " if \"i don't know\" in answer.lower() or \"don't know\" in answer.lower():\n", + " conf = 0.8 # High confidence in abstention\n", + "\n", + " # Penalty for very long answers (more likely to hallucinate)\n", + " if len(answer.split()) > 100:\n", + " conf *= 0.9\n", + "\n", + " # Penalty for very short answers\n", + " if len(answer.split()) < 3:\n", + " conf *= 0.7\n", + "\n", + " return min(max(conf, 0.0), 1.0)\n", + "\n", + "\n", + "# Initialize agents\n", + "logger_agent = LoggingAgent()\n", + "planner_agent = PlannerAgent(logger_agent)\n", + "executor_agent = ExecutorAgent(retrieval_system, available_models, logger_agent)\n", + "critic_agent = CriticAgent(logger_agent)\n", + "\n", + "print(\" \u2713 Agent system initialized:\")\n", + "print(\" - PlannerAgent (rule-based step planning)\")\n", + "print(\" - ExecutorAgent (retrieval + generation)\")\n", + "print(\" - CriticAgent (grounding + hallucination detection)\")\n", + "print(\" - LoggingAgent (structured activity recording)\")\n", + "print()\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 7: Tools\n", + "\n", + "Callable tool classes:\n", + "1. **RetrievalTool** \u2014 multi-strategy document retrieval\n", + "2. **VerificationTool** \u2014 answer grounding verification\n", + "3. **KeywordSearchTool** \u2014 simple keyword lookup in knowledge base" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# \u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n", + "# \u2551 SECTION 7: TOOLS \u2551\n", + "# \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n", + "\n", + "print(\"=\" * 70)\n", + "print(\"SECTION 7: Tools\")\n", + "print(\"=\" * 70)\n", + "\n", + "\n", + "class RetrievalTool:\n", + " \"\"\"Tool for document retrieval with configurable strategy.\"\"\"\n", + "\n", + " name = \"retrieval_tool\"\n", + " description = \"Retrieves relevant documents from the knowledge base.\"\n", + "\n", + " def __init__(self, retrieval_system: RetrievalSystem):\n", + " self.retrieval_system = retrieval_system\n", + "\n", + " def __call__(self, query: str, strategy: str = \"hybrid\", k: int = 5) -> List[Dict]:\n", + " \"\"\"Execute retrieval.\"\"\"\n", + " if strategy == \"hybrid\":\n", + " return self.retrieval_system.retrieve_hybrid(query, k)\n", + " elif strategy == \"dense\":\n", + " return self.retrieval_system.retrieve_dense(query, k)\n", + " elif strategy == \"two_stage\":\n", + " return self.retrieval_system.retrieve_two_stage(query, k)\n", + " return self.retrieval_system.retrieve_hybrid(query, k)\n", + "\n", + "\n", + "class VerificationTool:\n", + " \"\"\"Tool for verifying answer grounding against context.\"\"\"\n", + "\n", + " name = \"verification_tool\"\n", + " description = \"Verifies if an answer is grounded in the provided context.\"\n", + "\n", + " def __call__(self, answer: str, contexts: List[str]) -> Dict:\n", + " \"\"\"Verify answer grounding.\"\"\"\n", + " if not answer or not contexts:\n", + " return {\"verified\": False, \"reason\": \"Empty answer or context\"}\n", + "\n", + " # Token overlap check\n", + " answer_tokens = set(answer.lower().split())\n", + " context_tokens = set()\n", + " for ctx in contexts:\n", + " context_tokens.update(ctx.lower().split())\n", + "\n", + " stopwords = {\"the\", \"a\", \"an\", \"is\", \"are\", \"was\", \"were\", \"in\", \"on\", \"at\",\n", + " \"to\", \"for\", \"of\", \"and\", \"or\", \"it\", \"this\", \"that\", \"with\"}\n", + " answer_meaningful = answer_tokens - stopwords\n", + " context_meaningful = context_tokens - stopwords\n", + "\n", + " if not answer_meaningful:\n", + " return {\"verified\": False, \"reason\": \"No meaningful tokens in answer\"}\n", + "\n", + " overlap = len(answer_meaningful & context_meaningful) / len(answer_meaningful)\n", + " unsupported = answer_meaningful - context_meaningful\n", + "\n", + " return {\n", + " \"verified\": overlap > 0.5,\n", + " \"overlap_score\": round(overlap, 4),\n", + " \"unsupported_tokens\": list(unsupported)[:10],\n", + " \"reason\": \"Sufficient grounding\" if overlap > 0.5 else \"Insufficient grounding\",\n", + " }\n", + "\n", + "\n", + "class KeywordSearchTool:\n", + " \"\"\"Simple keyword search tool for quick lookups.\"\"\"\n", + "\n", + " name = \"keyword_search_tool\"\n", + " description = \"Searches for specific keywords in the knowledge base.\"\n", + "\n", + " def __init__(self, documents: List[str]):\n", + " self.documents = documents\n", + "\n", + " def __call__(self, keyword: str) -> List[Dict]:\n", + " \"\"\"Find documents containing the keyword.\"\"\"\n", + " results = []\n", + " keyword_lower = keyword.lower()\n", + " for i, doc in enumerate(self.documents):\n", + " if keyword_lower in doc.lower():\n", + " results.append({\n", + " \"index\": i,\n", + " \"document\": doc,\n", + " \"keyword_count\": doc.lower().count(keyword_lower),\n", + " })\n", + " return sorted(results, key=lambda x: x[\"keyword_count\"], reverse=True)[:5]\n", + "\n", + "\n", + "# Initialize tools\n", + "retrieval_tool = RetrievalTool(retrieval_system)\n", + "verification_tool = VerificationTool()\n", + "keyword_tool = KeywordSearchTool(knowledge_base)\n", + "\n", + "print(\" \u2713 Tools initialized:\")\n", + "print(\" - RetrievalTool (multi-strategy document retrieval)\")\n", + "print(\" - VerificationTool (answer grounding verification)\")\n", + "print(\" - KeywordSearchTool (simple keyword lookup)\")\n", + "print()\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 8: Hallucination Detection\n", + "\n", + "Multi-signal hallucination detector combining:\n", + "- Context overlap score (token-level)\n", + "- Grounding check (sentence-level)\n", + "- Semantic similarity (embedding-based)\n", + "- Factual consistency indicators (numbers, entities)\n", + "\n", + "**Classifications:** `faithful`, `factual`, `contextual`, `reasoning`\n", + "\n", + "Also computes a heuristic **confidence score** \u2208 [0, 1]." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# \u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n", + "# \u2551 SECTION 8: HALLUCINATION DETECTION \u2551\n", + "# \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n", + "\n", + "print(\"=\" * 70)\n", + "print(\"SECTION 8: Hallucination Detection\")\n", + "print(\"=\" * 70)\n", + "\n", + "\n", + "class HallucinationDetector:\n", + " \"\"\"\n", + " Multi-signal hallucination detector.\n", + " Combines context overlap, grounding checks, and optional LLM-as-judge.\n", + " \"\"\"\n", + "\n", + " def __init__(self, embedding_model: SentenceTransformer = None):\n", + " self.embedding_model = embedding_model\n", + "\n", + " def detect(self, answer: str, contexts: List[str], query: str = \"\") -> Dict:\n", + " \"\"\"\n", + " Run hallucination detection pipeline.\n", + " Returns classification, confidence, and detailed signals.\n", + " \"\"\"\n", + " signals = {}\n", + "\n", + " # Signal 1: Context overlap score\n", + " signals[\"context_overlap\"] = self._context_overlap(answer, contexts)\n", + "\n", + " # Signal 2: Grounding check (sentence-level)\n", + " signals[\"grounding\"] = self._grounding_check(answer, contexts)\n", + "\n", + " # Signal 3: Semantic similarity (if embedding model available)\n", + " if self.embedding_model:\n", + " signals[\"semantic_similarity\"] = self._semantic_similarity(answer, contexts)\n", + "\n", + " # Signal 4: Factual consistency indicators\n", + " signals[\"factual_indicators\"] = self._factual_indicators(answer, contexts)\n", + "\n", + " # Aggregate signals into classification\n", + " classification = self._classify(signals)\n", + "\n", + " return {\n", + " \"classification\": classification[\"type\"],\n", + " \"confidence\": classification[\"confidence\"],\n", + " \"signals\": signals,\n", + " \"is_hallucination\": classification[\"type\"] != \"faithful\",\n", + " }\n", + "\n", + " def _context_overlap(self, answer: str, contexts: List[str]) -> float:\n", + " \"\"\"Token-level overlap between answer and context.\"\"\"\n", + " if not answer or not contexts:\n", + " return 0.0\n", + "\n", + " answer_tokens = set(answer.lower().split())\n", + " context_tokens = set()\n", + " for ctx in contexts:\n", + " context_tokens.update(ctx.lower().split())\n", + "\n", + " stopwords = {\"the\", \"a\", \"an\", \"is\", \"are\", \"was\", \"were\", \"in\", \"on\", \"at\",\n", + " \"to\", \"for\", \"of\", \"and\", \"or\", \"it\", \"its\", \"this\", \"that\",\n", + " \"with\", \"by\", \"from\", \"as\", \"be\", \"has\", \"had\", \"have\"}\n", + " answer_meaningful = answer_tokens - stopwords\n", + " context_meaningful = context_tokens - stopwords\n", + "\n", + " if not answer_meaningful:\n", + " return 0.5 # Neutral if no meaningful tokens\n", + "\n", + " return len(answer_meaningful & context_meaningful) / len(answer_meaningful)\n", + "\n", + " def _grounding_check(self, answer: str, contexts: List[str]) -> Dict:\n", + " \"\"\"Sentence-level grounding assessment.\"\"\"\n", + " combined = \" \".join(contexts).lower()\n", + " sentences = [s.strip() for s in answer.split(\".\") if len(s.strip()) > 5]\n", + "\n", + " if not sentences:\n", + " return {\"grounded_ratio\": 0.0, \"total_sentences\": 0}\n", + "\n", + " grounded = 0\n", + " for sent in sentences:\n", + " words = set(sent.lower().split())\n", + " meaningful = words - {\"the\", \"a\", \"is\", \"are\", \"was\", \"it\", \"this\", \"that\", \"an\"}\n", + " if not meaningful:\n", + " grounded += 1\n", + " continue\n", + " ctx_words = set(combined.split())\n", + " ratio = len(meaningful & ctx_words) / len(meaningful)\n", + " if ratio > 0.4:\n", + " grounded += 1\n", + "\n", + " return {\n", + " \"grounded_ratio\": grounded / len(sentences),\n", + " \"total_sentences\": len(sentences),\n", + " \"grounded_sentences\": grounded,\n", + " }\n", + "\n", + " def _semantic_similarity(self, answer: str, contexts: List[str]) -> float:\n", + " \"\"\"Compute semantic similarity between answer and context.\"\"\"\n", + " if not self.embedding_model or not contexts:\n", + " return 0.0\n", + "\n", + " answer_emb = self.embedding_model.encode([answer], normalize_embeddings=True)\n", + " context_emb = self.embedding_model.encode(\n", + " [\" \".join(contexts)], normalize_embeddings=True\n", + " )\n", + " similarity = float(np.dot(answer_emb[0], context_emb[0]))\n", + " return similarity\n", + "\n", + " def _factual_indicators(self, answer: str, contexts: List[str]) -> Dict:\n", + " \"\"\"Check for factual consistency indicators.\"\"\"\n", + " answer_lower = answer.lower()\n", + " combined_context = \" \".join(contexts).lower()\n", + "\n", + " # Check for numbers in answer that aren't in context\n", + " import re\n", + " answer_numbers = set(re.findall(r'\\b\\d+\\.?\\d*\\b', answer_lower))\n", + " context_numbers = set(re.findall(r'\\b\\d+\\.?\\d*\\b', combined_context))\n", + " unsupported_numbers = answer_numbers - context_numbers\n", + "\n", + " # Check for proper nouns (capitalized words) not in context\n", + " answer_caps = set(re.findall(r'\\b[A-Z][a-z]+\\b', answer))\n", + " context_caps = set(re.findall(r'\\b[A-Z][a-z]+\\b', \" \".join(contexts)))\n", + " unsupported_entities = answer_caps - context_caps\n", + "\n", + " return {\n", + " \"unsupported_numbers\": list(unsupported_numbers),\n", + " \"unsupported_entities\": list(unsupported_entities)[:5],\n", + " \"has_unsupported_facts\": len(unsupported_numbers) > 0 or len(unsupported_entities) > 2,\n", + " }\n", + "\n", + " def _classify(self, signals: Dict) -> Dict:\n", + " \"\"\"Classify hallucination type based on aggregated signals.\"\"\"\n", + " overlap = signals.get(\"context_overlap\", 0)\n", + " grounding = signals.get(\"grounding\", {}).get(\"grounded_ratio\", 0)\n", + " semantic_sim = signals.get(\"semantic_similarity\", overlap) # fallback\n", + " has_unsupported = signals.get(\"factual_indicators\", {}).get(\"has_unsupported_facts\", False)\n", + "\n", + " # Weighted confidence\n", + " confidence = (0.3 * overlap + 0.3 * grounding + 0.2 * semantic_sim + 0.2 * (0.0 if has_unsupported else 1.0))\n", + "\n", + " # Classification logic\n", + " if confidence > 0.7:\n", + " return {\"type\": \"faithful\", \"confidence\": confidence}\n", + " elif has_unsupported and overlap < 0.4:\n", + " return {\"type\": \"factual\", \"confidence\": confidence}\n", + " elif grounding < 0.4:\n", + " return {\"type\": \"contextual\", \"confidence\": confidence}\n", + " else:\n", + " return {\"type\": \"reasoning\", \"confidence\": confidence}\n", + "\n", + "\n", + "# Initialize detector\n", + "hallucination_detector = HallucinationDetector(embedding_model=retrieval_system.embedding_model)\n", + "print(\" \u2713 Hallucination detector initialized\")\n", + "print(\" Signals: context_overlap, grounding_check, semantic_similarity, factual_indicators\")\n", + "print(\" Classifications: faithful, factual, contextual, reasoning\")\n", + "print()\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 9: Logging Agent Demo\n", + "\n", + "Demonstrates structured logging with `[PLANNER]`, `[EXECUTOR]`, `[CRITIC]`, `[LOG]` prefixes. \n", + "All logs stored in `logger_agent.get_logs()` for later evaluation." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# \u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n", + "# \u2551 SECTION 9: LOGGING AGENT (STRUCTURED LOGGING) \u2551\n", + "# \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n", + "\n", + "# LoggingAgent already defined in Section 6.\n", + "# Here we demonstrate its structured output.\n", + "\n", + "print(\"=\" * 70)\n", + "print(\"SECTION 9: Logging Agent Demo\")\n", + "print(\"=\" * 70)\n", + "print()\n", + "\n", + "# Quick demo of the logging system\n", + "logger_agent.log(\"LOG\", \"System initialized\", {\"models\": len(available_models), \"kb_size\": len(knowledge_base)})\n", + "logger_agent.log(\"LOG\", \"Ready for evaluation\")\n", + "print()\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 10: Dynamic Knowledge Update\n", + "\n", + "Demonstrates adding new documents to the knowledge base at runtime.\n", + "\n", + "The `add_new_document()` function:\n", + "- Chunks text into ~3-sentence segments\n", + "- Generates embeddings\n", + "- Updates FAISS index\n", + "- Rebuilds BM25 index\n", + "\n", + "> \u26a0\ufe0f **The model is NOT retrained. Only the retrieval knowledge base is updated.**" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# \u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n", + "# \u2551 SECTION 10: DYNAMIC KNOWLEDGE UPDATE \u2551\n", + "# \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n", + "\n", + "print(\"=\" * 70)\n", + "print(\"SECTION 10: Dynamic Knowledge Update\")\n", + "print(\"=\" * 70)\n", + "print()\n", + "print(\"NOTE: The model is NOT retrained. Only the retrieval knowledge base is updated.\")\n", + "print()\n", + "\n", + "\n", + "def add_new_document(text: str, retrieval_sys: RetrievalSystem = None):\n", + " \"\"\"\n", + " Add a new document to the knowledge base.\n", + " Chunks the text, generates embeddings, and updates both FAISS and BM25.\n", + "\n", + " NOTE: The model is not retrained. Only the retrieval knowledge base is updated.\n", + " \"\"\"\n", + " if retrieval_sys is None:\n", + " retrieval_sys = retrieval_system\n", + "\n", + " # Simple chunking: split by sentences, group into chunks of ~3 sentences\n", + " sentences = [s.strip() for s in text.replace(\"\\n\", \" \").split(\".\") if len(s.strip()) > 10]\n", + "\n", + " chunks = []\n", + " chunk_size = 3\n", + " for i in range(0, len(sentences), chunk_size):\n", + " chunk = \". \".join(sentences[i:i + chunk_size]) + \".\"\n", + " chunks.append(chunk)\n", + "\n", + " if not chunks:\n", + " chunks = [text]\n", + "\n", + " # Add each chunk to the retrieval system\n", + " for chunk in chunks:\n", + " retrieval_sys.add_document(chunk, metadata={\"source\": \"dynamic_update\"})\n", + "\n", + " print(f\" \u2713 Added {len(chunks)} chunk(s) to knowledge base\")\n", + " print(f\" \u2713 Knowledge base size: {len(retrieval_sys.documents)} documents\")\n", + " return chunks\n", + "\n", + "\n", + "# Demonstrate dynamic knowledge update\n", + "demo_text = \"\"\"\n", + "Large Language Models (LLMs) are neural networks with billions of parameters trained on massive\n", + "text corpora. Recent advances include mixture-of-experts architectures, which route different\n", + "inputs to specialized sub-networks. This allows scaling model capacity without proportionally\n", + "increasing compute costs. Models like Mixtral use 8 experts with a router that selects 2 experts\n", + "per token, achieving strong performance with lower inference cost than dense models of similar\n", + "total parameter count.\n", + "\"\"\"\n", + "\n", + "print(\" Adding new document about LLMs and Mixture of Experts...\")\n", + "new_chunks = add_new_document(demo_text)\n", + "print()\n", + "\n", + "# Test retrieval after update\n", + "test_results = retrieval_tool(\"What is mixture of experts in LLMs?\", strategy=\"hybrid\", k=3)\n", + "if test_results:\n", + " print(f\" \u2713 Verification: retrieved {len(test_results)} docs for MoE query\")\n", + " print(f\" Top result score: {test_results[0]['score']:.4f}\")\n", + "print()\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 11: Full Evaluation Loop\n", + "\n", + "Runs the complete pipeline over all 55 queries \u00d7 3 retrieval strategies \u00d7 3 models.\n", + "\n", + "Collects per-run: retrieval score, context overlap, answer length, hallucination type, confidence, latency." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# \u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n", + "# \u2551 SECTION 11: EVALUATION LOOP \u2551\n", + "# \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n", + "\n", + "print(\"=\" * 70)\n", + "print(\"SECTION 11: Full Evaluation Loop\")\n", + "print(\"=\" * 70)\n", + "print()\n", + "\n", + "# Clear logs for evaluation\n", + "logger_agent.clear()\n", + "\n", + "\n", + "def run_full_pipeline(\n", + " question: str,\n", + " retrieval_strategy: str,\n", + " model_tuple: tuple,\n", + " retrieval_sys: RetrievalSystem,\n", + " detector: HallucinationDetector,\n", + " logger: LoggingAgent,\n", + ") -> Dict:\n", + " \"\"\"\n", + " Run the complete RAG pipeline for a single query.\n", + " Returns all metrics for evaluation.\n", + " \"\"\"\n", + " start_time = time.time()\n", + " model_name, model = model_tuple\n", + "\n", + " # Step 1: Plan\n", + " plan = planner_agent.plan(question, retrieval_strategy)\n", + "\n", + " # Step 2: Retrieve\n", + " if retrieval_strategy == \"hybrid\":\n", + " retrieved = retrieval_sys.retrieve_hybrid(question, k=5)\n", + " elif retrieval_strategy == \"dense\":\n", + " retrieved = retrieval_sys.retrieve_dense(question, k=5)\n", + " elif retrieval_strategy == \"two_stage\":\n", + " retrieved = retrieval_sys.retrieve_two_stage(question, k=5)\n", + " else:\n", + " retrieved = retrieval_sys.retrieve_hybrid(question, k=5)\n", + "\n", + " contexts = [doc[\"document\"] for doc in retrieved]\n", + " retrieval_score = np.mean([doc[\"score\"] for doc in retrieved]) if retrieved else 0.0\n", + "\n", + " # Step 3: Generate\n", + " if hasattr(model, \"pipe\") and model.pipe and hasattr(model.pipe, \"tokenizer\"):\n", + " prompt = build_chat_prompt(question, contexts, model.pipe.tokenizer)\n", + " else:\n", + " prompt = build_rag_prompt(question, contexts)\n", + "\n", + " answer = model.generate(prompt, max_new_tokens=200)\n", + "\n", + " # Step 4: Detect hallucination\n", + " detection = detector.detect(answer, contexts, question)\n", + "\n", + " # Step 5: Compute metrics\n", + " total_latency = time.time() - start_time\n", + "\n", + " result = {\n", + " \"question\": question,\n", + " \"retrieval_strategy\": retrieval_strategy,\n", + " \"model_name\": model_name,\n", + " \"retrieved_docs\": contexts[:3], # Store top 3 for logging\n", + " \"retrieval_score\": retrieval_score,\n", + " \"generated_answer\": answer,\n", + " \"context_overlap\": detection[\"signals\"][\"context_overlap\"],\n", + " \"answer_length\": len(answer.split()),\n", + " \"hallucination_type\": detection[\"classification\"],\n", + " \"confidence_score\": detection[\"confidence\"],\n", + " \"is_hallucination\": detection[\"is_hallucination\"],\n", + " \"latency\": total_latency,\n", + " }\n", + "\n", + " # Log the result\n", + " logger.log(\"LOG\", f\"Pipeline complete: {retrieval_strategy}/{model_name}\", {\n", + " \"hallucination\": detection[\"classification\"],\n", + " \"confidence\": round(detection[\"confidence\"], 3),\n", + " \"latency_s\": round(total_latency, 3),\n", + " })\n", + "\n", + " return result\n", + "\n", + "\n", + "# Run evaluation\n", + "strategies = [\"hybrid\", \"dense\", \"two_stage\"]\n", + "all_results = []\n", + "\n", + "print(f\"Running evaluation: {len(eval_dataset)} queries \u00d7 {len(strategies)} strategies \u00d7 {len(available_models)} models\")\n", + "print(f\"Total runs: {len(eval_dataset) * len(strategies) * len(available_models)}\")\n", + "print()\n", + "\n", + "# Limit detailed logging during bulk evaluation\n", + "original_log = logger_agent.log\n", + "\n", + "\n", + "def quiet_log(agent, action, details=None):\n", + " \"\"\"Quiet logging during bulk eval (no print).\"\"\"\n", + " entry = {\n", + " \"timestamp\": time.time(),\n", + " \"agent\": agent,\n", + " \"action\": action,\n", + " \"details\": details or {},\n", + " }\n", + " logger_agent.logs.append(entry)\n", + "\n", + "\n", + "logger_agent.log = quiet_log # Suppress prints during bulk eval\n", + "\n", + "eval_count = 0\n", + "total_runs = len(eval_dataset) * len(strategies) * len(available_models)\n", + "\n", + "for strategy in strategies:\n", + " for model_tuple in available_models:\n", + " model_name = model_tuple[0]\n", + " for item in eval_dataset:\n", + " result = run_full_pipeline(\n", + " question=item[\"question\"],\n", + " retrieval_strategy=strategy,\n", + " model_tuple=model_tuple,\n", + " retrieval_sys=retrieval_system,\n", + " detector=hallucination_detector,\n", + " logger=logger_agent,\n", + " )\n", + " result[\"test_id\"] = item[\"id\"]\n", + " result[\"domain\"] = item[\"domain\"]\n", + " result[\"difficulty\"] = item[\"difficulty\"]\n", + " result[\"expected_hallucination_type\"] = item[\"expected_hallucination_type\"]\n", + " all_results.append(result)\n", + "\n", + " eval_count += 1\n", + " if eval_count % 50 == 0:\n", + " print(f\" Progress: {eval_count}/{total_runs} ({100*eval_count/total_runs:.0f}%)\")\n", + "\n", + "# Restore logging\n", + "logger_agent.log = original_log\n", + "\n", + "print(f\"\\n \u2713 Evaluation complete: {len(all_results)} total results\")\n", + "print()\n", + "\n", + "# Convert to DataFrame for analysis\n", + "results_df = pd.DataFrame(all_results)\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 12: Evaluation Plots (Matplotlib)\n", + "\n", + "6 plots generated:\n", + "1. Hallucination type distribution\n", + "2. Confidence score histogram\n", + "3. Retrieval strategy vs hallucination rate\n", + "4. Retrieval strategy vs latency\n", + "5. Model comparison (hallucination rate)\n", + "6. Average confidence by domain" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# \u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n", + "# \u2551 SECTION 12: PLOTS (MATPLOTLIB ONLY) \u2551\n", + "# \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n", + "\n", + "print(\"=\" * 70)\n", + "print(\"SECTION 12: Generating Plots\")\n", + "print(\"=\" * 70)\n", + "print()\n", + "\n", + "fig, axes = plt.subplots(3, 2, figsize=(14, 16))\n", + "fig.suptitle(\"HALT-RAG: Evaluation Results\", fontsize=16, fontweight=\"bold\", y=0.98)\n", + "\n", + "# Plot 1: Hallucination Type Distribution\n", + "ax = axes[0, 0]\n", + "hall_counts = results_df[\"hallucination_type\"].value_counts()\n", + "ax.bar(hall_counts.index, hall_counts.values)\n", + "ax.set_title(\"1. Hallucination Type Distribution\")\n", + "ax.set_xlabel(\"Hallucination Type\")\n", + "ax.set_ylabel(\"Count\")\n", + "ax.tick_params(axis=\"x\", rotation=15)\n", + "for i, (label, count) in enumerate(zip(hall_counts.index, hall_counts.values)):\n", + " ax.text(i, count + 0.5, str(count), ha=\"center\", fontsize=9)\n", + "\n", + "# Plot 2: Confidence Score Histogram\n", + "ax = axes[0, 1]\n", + "ax.hist(results_df[\"confidence_score\"], bins=20, edgecolor=\"black\", alpha=0.7)\n", + "ax.set_title(\"2. Confidence Score Distribution\")\n", + "ax.set_xlabel(\"Confidence Score\")\n", + "ax.set_ylabel(\"Frequency\")\n", + "ax.axvline(results_df[\"confidence_score\"].mean(), color=\"red\", linestyle=\"--\",\n", + " label=f'Mean: {results_df[\"confidence_score\"].mean():.3f}')\n", + "ax.legend()\n", + "\n", + "# Plot 3: Retrieval Strategy vs Hallucination Rate\n", + "ax = axes[1, 0]\n", + "hall_rate_by_strategy = results_df.groupby(\"retrieval_strategy\")[\"is_hallucination\"].mean()\n", + "ax.bar(hall_rate_by_strategy.index, hall_rate_by_strategy.values)\n", + "ax.set_title(\"3. Retrieval Strategy vs Hallucination Rate\")\n", + "ax.set_xlabel(\"Strategy\")\n", + "ax.set_ylabel(\"Hallucination Rate\")\n", + "ax.set_ylim(0, 1)\n", + "for i, (label, rate) in enumerate(zip(hall_rate_by_strategy.index, hall_rate_by_strategy.values)):\n", + " ax.text(i, rate + 0.02, f\"{rate:.2%}\", ha=\"center\", fontsize=9)\n", + "\n", + "# Plot 4: Retrieval Strategy vs Latency\n", + "ax = axes[1, 1]\n", + "latency_by_strategy = results_df.groupby(\"retrieval_strategy\")[\"latency\"].agg([\"mean\", \"std\"])\n", + "ax.bar(latency_by_strategy.index, latency_by_strategy[\"mean\"],\n", + " yerr=latency_by_strategy[\"std\"], capsize=5)\n", + "ax.set_title(\"4. Retrieval Strategy vs Latency\")\n", + "ax.set_xlabel(\"Strategy\")\n", + "ax.set_ylabel(\"Latency (seconds)\")\n", + "for i, (label, row) in enumerate(latency_by_strategy.iterrows()):\n", + " ax.text(i, row[\"mean\"] + row[\"std\"] + 0.01, f'{row[\"mean\"]:.3f}s', ha=\"center\", fontsize=9)\n", + "\n", + "# Plot 5: Model Comparison (Hallucination Rate)\n", + "ax = axes[2, 0]\n", + "hall_rate_by_model = results_df.groupby(\"model_name\")[\"is_hallucination\"].mean()\n", + "ax.barh(hall_rate_by_model.index, hall_rate_by_model.values)\n", + "ax.set_title(\"5. Model Comparison: Hallucination Rate\")\n", + "ax.set_xlabel(\"Hallucination Rate\")\n", + "ax.set_xlim(0, 1)\n", + "for i, (label, rate) in enumerate(zip(hall_rate_by_model.index, hall_rate_by_model.values)):\n", + " ax.text(rate + 0.02, i, f\"{rate:.2%}\", va=\"center\", fontsize=9)\n", + "\n", + "# Plot 6 (Bonus): Domain-level Performance\n", + "ax = axes[2, 1]\n", + "domain_confidence = results_df.groupby(\"domain\")[\"confidence_score\"].mean().sort_values()\n", + "ax.barh(domain_confidence.index, domain_confidence.values)\n", + "ax.set_title(\"6. Average Confidence by Domain\")\n", + "ax.set_xlabel(\"Mean Confidence Score\")\n", + "for i, (label, conf) in enumerate(zip(domain_confidence.index, domain_confidence.values)):\n", + " ax.text(conf + 0.01, i, f\"{conf:.3f}\", va=\"center\", fontsize=9)\n", + "\n", + "plt.tight_layout(rect=[0, 0, 1, 0.96])\n", + "plt.savefig(\"halt_rag_results.png\", dpi=150, bbox_inches=\"tight\")\n", + "plt.show()\n", + "print(\" \u2713 Plots saved to 'halt_rag_results.png'\")\n", + "print()\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 13: Final Summary & Analysis\n", + "\n", + "Prints:\n", + "- Best retrieval strategy (lowest hallucination rate)\n", + "- Best model\n", + "- Key observations\n", + "- Explicit limitations and caveats" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# \u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n", + "# \u2551 SECTION 13: OUTPUT + SUMMARY \u2551\n", + "# \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n", + "\n", + "print(\"=\" * 70)\n", + "print(\"SECTION 13: Final Summary & Analysis\")\n", + "print(\"=\" * 70)\n", + "print()\n", + "\n", + "# \u2500\u2500 Best Retrieval Strategy \u2500\u2500\n", + "print(\"\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\")\n", + "print(\"\u2502 RESULTS SUMMARY \u2502\")\n", + "print(\"\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\")\n", + "\n", + "# Strategy analysis\n", + "strategy_metrics = results_df.groupby(\"retrieval_strategy\").agg({\n", + " \"is_hallucination\": \"mean\",\n", + " \"confidence_score\": \"mean\",\n", + " \"latency\": \"mean\",\n", + " \"retrieval_score\": \"mean\",\n", + "}).round(4)\n", + "\n", + "print(\"\u2502\")\n", + "print(\"\u2502 Retrieval Strategy Comparison:\")\n", + "print(\"\u2502 \" + \"-\" * 67)\n", + "print(f\"\u2502 {'Strategy':<15} {'Hall. Rate':<12} {'Confidence':<12} {'Latency(s)':<12} {'Retr. Score':<12}\")\n", + "print(\"\u2502 \" + \"-\" * 67)\n", + "for strategy in strategy_metrics.index:\n", + " row = strategy_metrics.loc[strategy]\n", + " print(f\"\u2502 {strategy:<15} {row['is_hallucination']:<12.4f} {row['confidence_score']:<12.4f} \"\n", + " f\"{row['latency']:<12.4f} {row['retrieval_score']:<12.4f}\")\n", + "print(\"\u2502\")\n", + "\n", + "best_strategy = strategy_metrics[\"is_hallucination\"].idxmin()\n", + "print(f\"\u2502 \u2605 Best Retrieval Strategy: {best_strategy}\")\n", + "print(f\"\u2502 (lowest hallucination rate: {strategy_metrics.loc[best_strategy, 'is_hallucination']:.2%})\")\n", + "print(\"\u2502\")\n", + "\n", + "# Model analysis\n", + "model_metrics = results_df.groupby(\"model_name\").agg({\n", + " \"is_hallucination\": \"mean\",\n", + " \"confidence_score\": \"mean\",\n", + " \"latency\": \"mean\",\n", + "}).round(4)\n", + "\n", + "print(\"\u2502 Model Comparison:\")\n", + "print(\"\u2502 \" + \"-\" * 67)\n", + "print(f\"\u2502 {'Model':<25} {'Hall. Rate':<12} {'Confidence':<12} {'Latency(s)':<12}\")\n", + "print(\"\u2502 \" + \"-\" * 67)\n", + "for model in model_metrics.index:\n", + " row = model_metrics.loc[model]\n", + " print(f\"\u2502 {model:<25} {row['is_hallucination']:<12.4f} {row['confidence_score']:<12.4f} \"\n", + " f\"{row['latency']:<12.4f}\")\n", + "print(\"\u2502\")\n", + "\n", + "best_model = model_metrics[\"is_hallucination\"].idxmin()\n", + "print(f\"\u2502 \u2605 Best Model: {best_model}\")\n", + "print(f\"\u2502 (lowest hallucination rate: {model_metrics.loc[best_model, 'is_hallucination']:.2%})\")\n", + "print(\"\u2502\")\n", + "\n", + "# Key Observations\n", + "print(\"\u2502 KEY OBSERVATIONS:\")\n", + "print(\"\u2502 \" + \"-\" * 67)\n", + "\n", + "avg_confidence = results_df[\"confidence_score\"].mean()\n", + "print(f\"\u2502 \u2022 Average confidence score across all runs: {avg_confidence:.4f}\")\n", + "\n", + "hall_dist = results_df[\"hallucination_type\"].value_counts(normalize=True)\n", + "most_common_hall = hall_dist.index[0]\n", + "print(f\"\u2502 \u2022 Most common classification: '{most_common_hall}' ({hall_dist.iloc[0]:.1%} of answers)\")\n", + "\n", + "avg_latency = results_df[\"latency\"].mean()\n", + "print(f\"\u2502 \u2022 Average pipeline latency: {avg_latency:.3f} seconds\")\n", + "\n", + "# Difficulty analysis\n", + "diff_hall = results_df.groupby(\"difficulty\")[\"is_hallucination\"].mean()\n", + "print(f\"\u2502 \u2022 Hallucination rate by difficulty:\")\n", + "for diff in [\"easy\", \"medium\", \"hard\"]:\n", + " if diff in diff_hall.index:\n", + " print(f\"\u2502 - {diff}: {diff_hall[diff]:.2%}\")\n", + "\n", + "print(\"\u2502\")\n", + "print(\"\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\")\n", + "print(\"\u2502 LIMITATIONS & CAVEATS \u2502\")\n", + "print(\"\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\")\n", + "print(\"\u2502 \u2502\")\n", + "print(\"\u2502 1. SYNTHETIC DATASET: All test cases are artificially constructed. \u2502\")\n", + "print(\"\u2502 Results may not generalize to real-world queries and documents. \u2502\")\n", + "print(\"\u2502 \u2502\")\n", + "print(\"\u2502 2. PROBABILISTIC DETECTION: Hallucination classification is \u2502\")\n", + "print(\"\u2502 heuristic-based (token overlap + semantic similarity). It is \u2502\")\n", + "print(\"\u2502 NOT ground truth \u2014 false positives and negatives will occur. \u2502\")\n", + "print(\"\u2502 \u2502\")\n", + "print(\"\u2502 3. MODEL BIAS: Small models (TinyLlama, DistilGPT2) have limited \u2502\")\n", + "print(\"\u2502 instruction-following ability. Results would differ significantly\u2502\")\n", + "print(\"\u2502 with larger models (7B+, 70B+). \u2502\")\n", + "print(\"\u2502 \u2502\")\n", + "print(\"\u2502 4. CONTROLLED RETRIEVAL: The knowledge base is derived from the \u2502\")\n", + "print(\"\u2502 test set itself. In production, retrieval noise would increase \u2502\")\n", + "print(\"\u2502 hallucination rates significantly. \u2502\")\n", + "print(\"\u2502 \u2502\")\n", + "print(\"\u2502 5. NO TRUE GROUND-TRUTH EVALUATION: Without human-annotated \u2502\")\n", + "print(\"\u2502 hallucination labels, we cannot compute true precision/recall \u2502\")\n", + "print(\"\u2502 for the detection system. \u2502\")\n", + "print(\"\u2502 \u2502\")\n", + "print(\"\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\")\n", + "print()\n", + "\n", + "# \u2500\u2500 Logging Summary \u2500\u2500\n", + "print(\"Logging Summary:\")\n", + "log_summary = logger_agent.get_summary()\n", + "print(f\" Total log entries: {log_summary['total_entries']}\")\n", + "print(f\" By agent: {dict(log_summary['by_agent'])}\")\n", + "print()\n", + "\n", + "# \u2500\u2500 Sample Logs \u2500\u2500\n", + "print(\"Sample Structured Logs (last 5):\")\n", + "for log_entry in logger_agent.get_logs()[-5:]:\n", + " ts = time.strftime(\"%H:%M:%S\", time.localtime(log_entry[\"timestamp\"]))\n", + " print(f\" [{ts}] [{log_entry['agent']:8s}] {log_entry['action']}\")\n", + "\n", + "print()\n", + "print(\"=\" * 70)\n", + "print(\"HALT-RAG Demo Complete\")\n", + "print(\"=\" * 70)\n", + "print()\n", + "print(\"Files generated:\")\n", + "print(\" \u2022 halt_rag_results.png \u2014 Evaluation plots\")\n", + "print()\n", + "print(\"To access full logs programmatically:\")\n", + "print(\" logs = logger_agent.get_logs()\")\n", + "print(\" results = results_df # pandas DataFrame with all evaluation results\")\n", + "print()\n", + "print(\"This is a graduate-level AI systems demo.\")\n", + "print(\"Clarity, correctness, and reproducibility > complexity.\")\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## \u2705 Demo Complete\n", + "\n", + "**Access results programmatically:**\n", + "```python\n", + "logs = logger_agent.get_logs() # All structured logs\n", + "results = results_df # pandas DataFrame with all evaluation results\n", + "```\n", + "\n", + "**This is a graduate-level AI systems demo.** \n", + "Clarity, correctness, and reproducibility > complexity." + ] + } + ] +} \ No newline at end of file