File size: 141,510 Bytes
9b88a42 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 | {
"nbformat": 4,
"nbformat_minor": 5,
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"name": "python",
"version": "3.10.12",
"mimetype": "text/x-python",
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"pygments_lexer": "ipython3"
},
"accelerator": "GPU",
"gpuClass": "premium"
},
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# \ud83d\udee1\ufe0f HALT-RAG: Hallucination-Aware Retrieval-Augmented Generation\n",
"\n",
"A complete research-style demo implementing:\n",
"- **Experimental retrieval strategies** (Hybrid BM25+FAISS, Dense-only, Two-stage rerank)\n",
"- **Lightweight agent workflow** (Planner \u2192 Executor \u2192 Critic \u2192 Logger)\n",
"- **Hallucination detection** with classification\n",
"- **Structured logging**\n",
"- **Evaluation + visualization**\n",
"\n",
"**Runtime:** Google Colab Pro (A100 GPU) \n",
"**Author:** Graduate AI Systems Project \n",
"**License:** MIT \n",
"\n",
"> \u26a0\ufe0f This uses a **SYNTHETIC** dataset for evaluation purposes only."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Section 1: Setup & Installation\n",
"\n",
"Install required packages and verify GPU availability."
]
},
{
"cell_type": "code",
"metadata": {},
"source": [
"# \u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n",
"# \u2551 SECTION 1: SETUP & INSTALLATION \u2551\n",
"# \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n",
"\n",
"!pip install -q sentence-transformers faiss-cpu rank-bm25 transformers accelerate matplotlib pandas numpy\n",
"\n",
"import time\n",
"import json\n",
"import re\n",
"import warnings\n",
"from dataclasses import dataclass, field, asdict\n",
"from typing import List, Dict, Tuple, Optional, Any\n",
"from collections import Counter\n",
"\n",
"import numpy as np\n",
"import pandas as pd\n",
"import matplotlib.pyplot as plt\n",
"import torch\n",
"import faiss\n",
"from rank_bm25 import BM25Okapi\n",
"from sentence_transformers import SentenceTransformer\n",
"from transformers import pipeline, AutoModelForCausalLM, AutoTokenizer\n",
"\n",
"warnings.filterwarnings(\"ignore\")\n",
"\n",
"print(\"=\" * 70)\n",
"print(\"HALT-RAG: Hallucination-Aware Retrieval-Augmented Generation\")\n",
"print(\"=\" * 70)\n",
"print(f\"PyTorch version: {torch.__version__}\")\n",
"print(f\"CUDA available: {torch.cuda.is_available()}\")\n",
"if torch.cuda.is_available():\n",
" print(f\"GPU: {torch.cuda.get_device_name(0)}\")\n",
"print()"
],
"outputs": [],
"execution_count": null
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Section 2: Synthetic Evaluation Dataset\n",
"\n",
"Create a synthetic evaluation dataset with 55 test cases across multiple domains.\n",
"\n",
"Each test case includes: `id`, `question`, `expected_answer`, `relevant_context`, `domain`, `difficulty`, `expected_hallucination_type`\n",
"\n",
"> **NOTE:** This dataset is SYNTHETIC \u2014 created for evaluation purposes only."
]
},
{
"cell_type": "code",
"metadata": {},
"source": [
"print(\"=\" * 70)\n",
"print(\"SECTION 2: Creating Synthetic Evaluation Dataset\")\n",
"print(\"=\" * 70)\n",
"print(\"NOTE: This dataset is SYNTHETIC \u2014 created for evaluation purposes only.\")\n",
"print()\n",
"\n",
"\n",
"def create_synthetic_dataset() -> List[Dict]:\n",
" \"\"\"\n",
" Create a synthetic evaluation dataset with 50+ test cases.\n",
" Each case is designed to test different aspects of RAG + hallucination detection.\n",
"\n",
" Domains: Science, History, Technology, Medicine, Geography\n",
" Difficulties: easy, medium, hard\n",
" Expected hallucination types: factual, contextual, reasoning, faithful\n",
" \"\"\"\n",
" dataset = [\n",
" # === SCIENCE DOMAIN (10 cases) ===\n",
" {\n",
" \"id\": \"SCI-001\",\n",
" \"question\": \"What is the speed of light in a vacuum?\",\n",
" \"expected_answer\": \"The speed of light in a vacuum is approximately 299,792,458 meters per second.\",\n",
" \"relevant_context\": \"Light travels at a constant speed in a vacuum, measured precisely at 299,792,458 meters per second (approximately 3 \u00d7 10^8 m/s). This is denoted by the constant 'c' and is a fundamental physical constant.\",\n",
" \"domain\": \"science\",\n",
" \"difficulty\": \"easy\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"SCI-002\",\n",
" \"question\": \"What causes the aurora borealis?\",\n",
" \"expected_answer\": \"The aurora borealis is caused by charged particles from the sun interacting with gases in Earth's atmosphere.\",\n",
" \"relevant_context\": \"The aurora borealis (Northern Lights) occurs when charged particles from the solar wind interact with gases in Earth's upper atmosphere. These particles are guided by Earth's magnetic field toward the poles, where they collide with oxygen and nitrogen atoms, causing them to emit light.\",\n",
" \"domain\": \"science\",\n",
" \"difficulty\": \"medium\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"SCI-003\",\n",
" \"question\": \"How does photosynthesis convert sunlight into energy?\",\n",
" \"expected_answer\": \"Photosynthesis uses chlorophyll to absorb sunlight, converting CO2 and water into glucose and oxygen through light-dependent and light-independent reactions.\",\n",
" \"relevant_context\": \"Photosynthesis occurs in two stages: light-dependent reactions in the thylakoid membranes capture solar energy using chlorophyll pigments, splitting water molecules and generating ATP and NADPH. The Calvin cycle (light-independent reactions) in the stroma then uses these energy carriers to fix CO2 into glucose (C6H12O6), releasing oxygen as a byproduct.\",\n",
" \"domain\": \"science\",\n",
" \"difficulty\": \"medium\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"SCI-004\",\n",
" \"question\": \"What is the Heisenberg uncertainty principle?\",\n",
" \"expected_answer\": \"The Heisenberg uncertainty principle states that it is impossible to simultaneously know both the exact position and exact momentum of a particle.\",\n",
" \"relevant_context\": \"The Heisenberg uncertainty principle, formulated by Werner Heisenberg in 1927, establishes a fundamental limit in quantum mechanics. It states that the more precisely the position of a particle is determined, the less precisely its momentum can be known, and vice versa. Mathematically: \u0394x\u00b7\u0394p \u2265 \u210f/2.\",\n",
" \"domain\": \"science\",\n",
" \"difficulty\": \"hard\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"SCI-005\",\n",
" \"question\": \"What temperature does water boil at on Mars?\",\n",
" \"expected_answer\": \"Water boils at a much lower temperature on Mars due to the very low atmospheric pressure, roughly around 0-10\u00b0C depending on conditions.\",\n",
" \"relevant_context\": \"Mars has an atmospheric pressure of about 610 Pascals (0.6% of Earth's sea level pressure). Due to this extremely low pressure, the boiling point of water on Mars is significantly reduced. At typical Martian surface pressures, water would boil at approximately 0-2\u00b0C rather than 100\u00b0C as on Earth at sea level.\",\n",
" \"domain\": \"science\",\n",
" \"difficulty\": \"hard\",\n",
" \"expected_hallucination_type\": \"contextual\",\n",
" },\n",
" {\n",
" \"id\": \"SCI-006\",\n",
" \"question\": \"What is CRISPR-Cas9?\",\n",
" \"expected_answer\": \"CRISPR-Cas9 is a gene-editing technology that allows precise modification of DNA sequences in living organisms.\",\n",
" \"relevant_context\": \"CRISPR-Cas9 is a revolutionary gene-editing tool adapted from a natural defense mechanism in bacteria. The system uses a guide RNA to direct the Cas9 enzyme to a specific location in the genome, where it makes a precise cut in the DNA double helix. This allows researchers to delete, modify, or insert genetic material at targeted locations.\",\n",
" \"domain\": \"science\",\n",
" \"difficulty\": \"medium\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"SCI-007\",\n",
" \"question\": \"How do black holes form?\",\n",
" \"expected_answer\": \"Black holes form when massive stars exhaust their nuclear fuel and collapse under their own gravity.\",\n",
" \"relevant_context\": \"Stellar black holes form through gravitational collapse when a massive star (typically >25 solar masses) exhausts its nuclear fuel. Without the outward radiation pressure to counteract gravity, the core collapses. If the remaining mass exceeds the Tolman-Oppenheimer-Volkoff limit (~2-3 solar masses), neutron degeneracy pressure cannot halt the collapse, and a black hole forms.\",\n",
" \"domain\": \"science\",\n",
" \"difficulty\": \"medium\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"SCI-008\",\n",
" \"question\": \"What is dark matter made of?\",\n",
" \"expected_answer\": \"The composition of dark matter is unknown. Leading candidates include WIMPs and axions, but it has not been directly detected.\",\n",
" \"relevant_context\": \"Dark matter constitutes approximately 27% of the universe's mass-energy content, yet its composition remains one of physics' greatest mysteries. It does not emit, absorb, or reflect electromagnetic radiation. Leading theoretical candidates include Weakly Interacting Massive Particles (WIMPs), axions, and sterile neutrinos. Despite numerous detection experiments, dark matter has not been directly observed.\",\n",
" \"domain\": \"science\",\n",
" \"difficulty\": \"hard\",\n",
" \"expected_hallucination_type\": \"reasoning\",\n",
" },\n",
" {\n",
" \"id\": \"SCI-009\",\n",
" \"question\": \"What is the half-life of carbon-14?\",\n",
" \"expected_answer\": \"The half-life of carbon-14 is approximately 5,730 years.\",\n",
" \"relevant_context\": \"Carbon-14 (14C) is a radioactive isotope of carbon with a half-life of 5,730 \u00b1 40 years. It is continuously produced in the upper atmosphere by cosmic ray bombardment of nitrogen-14. This predictable decay rate makes it invaluable for radiocarbon dating of organic materials up to about 50,000 years old.\",\n",
" \"domain\": \"science\",\n",
" \"difficulty\": \"easy\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"SCI-010\",\n",
" \"question\": \"Why is the sky blue?\",\n",
" \"expected_answer\": \"The sky appears blue because of Rayleigh scattering \u2014 shorter blue wavelengths of sunlight are scattered more than longer wavelengths by atmospheric molecules.\",\n",
" \"relevant_context\": \"The blue color of the sky is caused by Rayleigh scattering. When sunlight enters Earth's atmosphere, it collides with gas molecules. Since the scattering intensity is inversely proportional to the fourth power of wavelength (1/\u03bb\u2074), shorter wavelengths (blue/violet) scatter much more than longer wavelengths (red/orange). Our eyes are more sensitive to blue than violet, so we perceive the sky as blue.\",\n",
" \"domain\": \"science\",\n",
" \"difficulty\": \"easy\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" # === HISTORY DOMAIN (10 cases) ===\n",
" {\n",
" \"id\": \"HIS-001\",\n",
" \"question\": \"When did World War II end?\",\n",
" \"expected_answer\": \"World War II ended in 1945, with Germany surrendering in May and Japan in August/September.\",\n",
" \"relevant_context\": \"World War II ended in 1945. Germany surrendered unconditionally on May 8, 1945 (V-E Day). Japan announced its surrender on August 15, 1945 (V-J Day), following the atomic bombings of Hiroshima and Nagasaki. The formal surrender was signed on September 2, 1945, aboard the USS Missouri in Tokyo Bay.\",\n",
" \"domain\": \"history\",\n",
" \"difficulty\": \"easy\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"HIS-002\",\n",
" \"question\": \"What caused the fall of the Roman Empire?\",\n",
" \"expected_answer\": \"The fall of the Roman Empire was caused by multiple factors including economic troubles, military overextension, barbarian invasions, and political instability.\",\n",
" \"relevant_context\": \"Historians attribute the fall of the Western Roman Empire (476 CE) to a complex combination of factors: severe economic troubles and overtaxation, military overextension and reliance on mercenaries, repeated barbarian invasions (Visigoths, Vandals, Huns), political instability with rapid emperor turnover, administrative division, and loss of traditional civic values.\",\n",
" \"domain\": \"history\",\n",
" \"difficulty\": \"medium\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"HIS-003\",\n",
" \"question\": \"Who built the Great Wall of China?\",\n",
" \"expected_answer\": \"The Great Wall was built over many centuries by multiple Chinese dynasties, with the most well-known sections constructed during the Ming Dynasty (1368-1644).\",\n",
" \"relevant_context\": \"The Great Wall of China was not built by a single ruler but constructed and reconstructed over approximately 2,000 years by multiple dynasties. The earliest walls date to the 7th century BCE. Emperor Qin Shi Huang connected existing walls around 221 BCE. The most recognizable sections were built during the Ming Dynasty (1368-1644 CE), stretching over 8,850 kilometers.\",\n",
" \"domain\": \"history\",\n",
" \"difficulty\": \"medium\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"HIS-004\",\n",
" \"question\": \"What was the significance of the Rosetta Stone?\",\n",
" \"expected_answer\": \"The Rosetta Stone was key to deciphering Egyptian hieroglyphics because it contained the same text in three scripts: hieroglyphic, Demotic, and Greek.\",\n",
" \"relevant_context\": \"The Rosetta Stone, discovered in 1799 by French soldiers in Egypt, contains a decree issued in 196 BCE written in three scripts: Ancient Egyptian hieroglyphics, Demotic script, and Ancient Greek. Since scholars could read Greek, they could use it as a reference to decode the hieroglyphics. Jean-Fran\u00e7ois Champollion finally deciphered the hieroglyphics in 1822.\",\n",
" \"domain\": \"history\",\n",
" \"difficulty\": \"medium\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"HIS-005\",\n",
" \"question\": \"What year did the first humans land on the Moon?\",\n",
" \"expected_answer\": \"The first humans landed on the Moon in 1969 during the Apollo 11 mission.\",\n",
" \"relevant_context\": \"On July 20, 1969, NASA's Apollo 11 mission successfully landed the first humans on the Moon. Astronauts Neil Armstrong and Buzz Aldrin walked on the lunar surface while Michael Collins orbited above in the command module. Armstrong's famous words upon stepping onto the surface were 'That's one small step for man, one giant leap for mankind.'\",\n",
" \"domain\": \"history\",\n",
" \"difficulty\": \"easy\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"HIS-006\",\n",
" \"question\": \"What were the main causes of World War I?\",\n",
" \"expected_answer\": \"The main causes of WWI include militarism, alliances, imperialism, nationalism, and the assassination of Archduke Franz Ferdinand.\",\n",
" \"relevant_context\": \"The causes of World War I are often summarized by the acronym MANIA: Militarism (arms race between European powers), Alliances (complex treaty obligations), Nationalism (ethnic tensions especially in the Balkans), Imperialism (competition for colonies), and the immediate trigger: the Assassination of Archduke Franz Ferdinand of Austria-Hungary on June 28, 1914, in Sarajevo.\",\n",
" \"domain\": \"history\",\n",
" \"difficulty\": \"medium\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"HIS-007\",\n",
" \"question\": \"How did the Black Death spread across Europe?\",\n",
" \"expected_answer\": \"The Black Death spread primarily through fleas on rats, traveling along trade routes from Central Asia to Europe via merchant ships.\",\n",
" \"relevant_context\": \"The Black Death (1347-1351) spread from Central Asia along the Silk Road trade routes. It reached Europe via merchant ships, notably arriving at the Sicilian port of Messina in October 1347. The plague was caused by Yersinia pestis bacteria, primarily transmitted through bites of infected fleas living on black rats. Crowded medieval cities with poor sanitation facilitated rapid spread, killing 30-60% of Europe's population.\",\n",
" \"domain\": \"history\",\n",
" \"difficulty\": \"medium\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"HIS-008\",\n",
" \"question\": \"What was the exact population of ancient Rome at its peak?\",\n",
" \"expected_answer\": \"The exact population is uncertain, but estimates suggest Rome had approximately 1 million inhabitants at its peak around 100-200 CE.\",\n",
" \"relevant_context\": \"The population of ancient Rome at its peak is debated among historians. Most estimates suggest the city housed between 800,000 and 1.2 million people during the 1st-2nd centuries CE. Census records exist but are incomplete and their interpretation is contested. The broader Roman Empire may have contained 55-70 million people at its height.\",\n",
" \"domain\": \"history\",\n",
" \"difficulty\": \"hard\",\n",
" \"expected_hallucination_type\": \"reasoning\",\n",
" },\n",
" {\n",
" \"id\": \"HIS-009\",\n",
" \"question\": \"Who invented the printing press?\",\n",
" \"expected_answer\": \"Johannes Gutenberg invented the movable-type printing press around 1440 in Mainz, Germany.\",\n",
" \"relevant_context\": \"Johannes Gutenberg is credited with inventing the movable-type printing press in Europe around 1440 in Mainz, Germany. His key innovation was the combination of a hand mold for casting metal type, oil-based ink, and a wooden press adapted from wine/olive presses. His most famous work, the Gutenberg Bible (1455), demonstrated the technology's potential. Note: movable type existed earlier in China (Bi Sheng, 1040 CE) and Korea.\",\n",
" \"domain\": \"history\",\n",
" \"difficulty\": \"easy\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"HIS-010\",\n",
" \"question\": \"What happened at the Battle of Thermopylae?\",\n",
" \"expected_answer\": \"At Thermopylae in 480 BCE, a small Greek force led by King Leonidas and 300 Spartans held the narrow pass against the massive Persian army of Xerxes for three days.\",\n",
" \"relevant_context\": \"The Battle of Thermopylae (480 BCE) was a famous last stand. King Leonidas I of Sparta, with approximately 300 Spartans and several thousand other Greek allies (estimates of 5,000-7,000 total), held the narrow coastal pass of Thermopylae against Xerxes I's Persian army (estimated 70,000-300,000). They held for three days before being outflanked via a mountain path revealed by a local traitor named Ephialtes.\",\n",
" \"domain\": \"history\",\n",
" \"difficulty\": \"hard\",\n",
" \"expected_hallucination_type\": \"contextual\",\n",
" },\n",
" # === TECHNOLOGY DOMAIN (10 cases) ===\n",
" {\n",
" \"id\": \"TECH-001\",\n",
" \"question\": \"What is a transformer model in machine learning?\",\n",
" \"expected_answer\": \"A transformer is a neural network architecture that uses self-attention mechanisms to process sequential data in parallel, introduced in the 'Attention Is All You Need' paper.\",\n",
" \"relevant_context\": \"The Transformer architecture was introduced in the 2017 paper 'Attention Is All You Need' by Vaswani et al. at Google. It uses self-attention mechanisms to process all positions in a sequence simultaneously (in parallel), unlike recurrent models (RNNs/LSTMs) that process sequentially. Key components include multi-head attention, positional encoding, and feed-forward layers. Transformers form the basis of models like BERT, GPT, and T5.\",\n",
" \"domain\": \"technology\",\n",
" \"difficulty\": \"medium\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"TECH-002\",\n",
" \"question\": \"How does blockchain work?\",\n",
" \"expected_answer\": \"Blockchain is a distributed ledger that records transactions in linked, cryptographically secured blocks. Each block contains a hash of the previous block, creating an immutable chain.\",\n",
" \"relevant_context\": \"A blockchain is a decentralized, distributed digital ledger. Transactions are grouped into blocks, each containing: a cryptographic hash of the previous block, a timestamp, and transaction data. This linking creates a chain where altering any block would invalidate all subsequent blocks. Consensus mechanisms (Proof of Work, Proof of Stake) ensure agreement among network participants without a central authority.\",\n",
" \"domain\": \"technology\",\n",
" \"difficulty\": \"medium\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"TECH-003\",\n",
" \"question\": \"What is quantum computing?\",\n",
" \"expected_answer\": \"Quantum computing uses quantum mechanical phenomena like superposition and entanglement to perform computations that are intractable for classical computers.\",\n",
" \"relevant_context\": \"Quantum computing harnesses quantum mechanical phenomena to process information. Unlike classical bits (0 or 1), quantum bits (qubits) can exist in superposition (both states simultaneously). Entanglement allows correlated qubits to be linked regardless of distance. These properties enable quantum computers to solve certain problems exponentially faster than classical computers, such as factoring large numbers (Shor's algorithm) and searching unsorted databases (Grover's algorithm).\",\n",
" \"domain\": \"technology\",\n",
" \"difficulty\": \"hard\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"TECH-004\",\n",
" \"question\": \"When was the first iPhone released?\",\n",
" \"expected_answer\": \"The first iPhone was released on June 29, 2007.\",\n",
" \"relevant_context\": \"The original iPhone was announced by Steve Jobs on January 9, 2007, at the Macworld Conference & Expo, and released on June 29, 2007. It featured a 3.5-inch touchscreen, 2-megapixel camera, and ran iPhone OS 1.0. It was initially exclusive to AT&T in the United States and priced at $499 for the 4GB model and $599 for 8GB.\",\n",
" \"domain\": \"technology\",\n",
" \"difficulty\": \"easy\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"TECH-005\",\n",
" \"question\": \"What is the difference between supervised and unsupervised learning?\",\n",
" \"expected_answer\": \"Supervised learning uses labeled data to train models to predict outcomes, while unsupervised learning finds patterns in unlabeled data without predefined targets.\",\n",
" \"relevant_context\": \"In machine learning, supervised learning trains models on labeled datasets where each example has an input-output pair. The model learns to map inputs to correct outputs (classification, regression). Unsupervised learning works with unlabeled data, discovering hidden patterns, groupings, or structure (clustering, dimensionality reduction, anomaly detection). Semi-supervised learning combines both approaches.\",\n",
" \"domain\": \"technology\",\n",
" \"difficulty\": \"easy\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"TECH-006\",\n",
" \"question\": \"How does GPT-4 compare to GPT-3 in terms of parameters?\",\n",
" \"expected_answer\": \"The exact parameter count of GPT-4 has not been officially disclosed by OpenAI, unlike GPT-3 which has 175 billion parameters.\",\n",
" \"relevant_context\": \"GPT-3, released in 2020, contains 175 billion parameters. OpenAI has not officially disclosed the parameter count of GPT-4 (released March 2023). Various rumors and leaks suggest it may use a Mixture of Experts architecture with approximately 1.7 trillion total parameters across 16 experts, but this has not been confirmed by OpenAI. The company has moved away from disclosing model sizes.\",\n",
" \"domain\": \"technology\",\n",
" \"difficulty\": \"hard\",\n",
" \"expected_hallucination_type\": \"reasoning\",\n",
" },\n",
" {\n",
" \"id\": \"TECH-007\",\n",
" \"question\": \"What programming language is Python named after?\",\n",
" \"expected_answer\": \"Python is named after the BBC comedy series 'Monty Python's Flying Circus', not the snake.\",\n",
" \"relevant_context\": \"Python was created by Guido van Rossum and first released in 1991. The language is named after the British comedy group Monty Python (specifically their show 'Monty Python's Flying Circus'), not after the snake. Van Rossum was reading scripts from the show while developing the language and wanted a short, unique, and slightly mysterious name.\",\n",
" \"domain\": \"technology\",\n",
" \"difficulty\": \"easy\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"TECH-008\",\n",
" \"question\": \"What is federated learning?\",\n",
" \"expected_answer\": \"Federated learning is a machine learning approach where models are trained across decentralized devices holding local data, without exchanging raw data.\",\n",
" \"relevant_context\": \"Federated learning is a distributed machine learning approach proposed by Google in 2016. Instead of centralizing training data, the model is sent to devices (phones, hospitals, etc.) where it trains on local data. Only model updates (gradients) are sent back to a central server for aggregation. This preserves data privacy as raw data never leaves the device. Challenges include non-IID data distributions and communication efficiency.\",\n",
" \"domain\": \"technology\",\n",
" \"difficulty\": \"hard\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"TECH-009\",\n",
" \"question\": \"How many transistors are in a modern CPU?\",\n",
" \"expected_answer\": \"Modern CPUs contain tens of billions of transistors. For example, Apple's M2 Ultra has about 134 billion transistors.\",\n",
" \"relevant_context\": \"Modern processors contain billions of transistors. As of 2023-2024, high-end examples include: Apple M2 Ultra (~134 billion transistors), AMD EPYC Genoa (~90 billion), Intel's Ponte Vecchio GPU (~100 billion). Consumer CPUs like AMD Ryzen 9 7950X contain about 13 billion transistors. Transistor counts have followed Moore's Law for decades, roughly doubling every 2 years.\",\n",
" \"domain\": \"technology\",\n",
" \"difficulty\": \"medium\",\n",
" \"expected_hallucination_type\": \"factual\",\n",
" },\n",
" {\n",
" \"id\": \"TECH-010\",\n",
" \"question\": \"What is the halting problem?\",\n",
" \"expected_answer\": \"The halting problem proves that no general algorithm can determine whether an arbitrary program will eventually halt or run forever.\",\n",
" \"relevant_context\": \"The halting problem, proven undecidable by Alan Turing in 1936, states that no general algorithm can determine, for all possible program-input pairs, whether the program will eventually halt (finish) or continue to run forever. This was proven by contradiction: assuming such an algorithm exists leads to a logical paradox. It establishes fundamental limits on what computers can decide.\",\n",
" \"domain\": \"technology\",\n",
" \"difficulty\": \"hard\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" # === MEDICINE DOMAIN (10 cases) ===\n",
" {\n",
" \"id\": \"MED-001\",\n",
" \"question\": \"What is the function of insulin in the human body?\",\n",
" \"expected_answer\": \"Insulin regulates blood glucose levels by enabling cells to absorb glucose from the bloodstream for energy or storage.\",\n",
" \"relevant_context\": \"Insulin is a peptide hormone produced by beta cells in the pancreatic islets of Langerhans. Its primary function is to regulate blood glucose levels. When blood sugar rises (e.g., after eating), insulin signals cells (especially muscle, fat, and liver cells) to absorb glucose from the bloodstream. It promotes glycogen synthesis in the liver and fat storage in adipose tissue, effectively lowering blood sugar levels.\",\n",
" \"domain\": \"medicine\",\n",
" \"difficulty\": \"easy\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"MED-002\",\n",
" \"question\": \"How do mRNA vaccines work?\",\n",
" \"expected_answer\": \"mRNA vaccines deliver synthetic mRNA that instructs cells to produce a harmless piece of the target virus (like the spike protein), triggering an immune response.\",\n",
" \"relevant_context\": \"mRNA vaccines contain synthetic messenger RNA encoding instructions for producing a specific viral protein (e.g., SARS-CoV-2 spike protein). Once injected, the mRNA enters cells and is translated by ribosomes into the target protein. The immune system recognizes this protein as foreign, producing antibodies and training T-cells. The mRNA is naturally degraded within days and does not alter DNA, which remains in the nucleus.\",\n",
" \"domain\": \"medicine\",\n",
" \"difficulty\": \"medium\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"MED-003\",\n",
" \"question\": \"What is the difference between Type 1 and Type 2 diabetes?\",\n",
" \"expected_answer\": \"Type 1 diabetes is an autoimmune condition where the body destroys insulin-producing cells. Type 2 is characterized by insulin resistance, often linked to lifestyle factors.\",\n",
" \"relevant_context\": \"Type 1 diabetes is an autoimmune disease where the immune system destroys pancreatic beta cells, eliminating insulin production. It typically appears in childhood/adolescence and requires lifelong insulin therapy. Type 2 diabetes involves insulin resistance \u2014 cells don't respond effectively to insulin \u2014 often combined with gradually declining insulin production. It accounts for ~90% of diabetes cases and is associated with obesity, sedentary lifestyle, and genetic factors.\",\n",
" \"domain\": \"medicine\",\n",
" \"difficulty\": \"medium\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"MED-004\",\n",
" \"question\": \"Can aspirin cure cancer?\",\n",
" \"expected_answer\": \"Aspirin cannot cure cancer. Some studies suggest it may reduce risk of certain cancers (particularly colorectal), but it is not a cancer treatment.\",\n",
" \"relevant_context\": \"Aspirin (acetylsalicylic acid) is a nonsteroidal anti-inflammatory drug (NSAID). Research has shown associations between regular low-dose aspirin use and reduced risk of colorectal cancer (approximately 20-30% reduction in some studies). However, aspirin is NOT a cancer treatment or cure. Its anti-cancer mechanisms likely involve COX-2 inhibition reducing inflammation. Current medical guidance does not recommend aspirin solely for cancer prevention due to bleeding risks.\",\n",
" \"domain\": \"medicine\",\n",
" \"difficulty\": \"hard\",\n",
" \"expected_hallucination_type\": \"factual\",\n",
" },\n",
" {\n",
" \"id\": \"MED-005\",\n",
" \"question\": \"What is the normal resting heart rate for adults?\",\n",
" \"expected_answer\": \"The normal resting heart rate for adults is between 60 and 100 beats per minute.\",\n",
" \"relevant_context\": \"A normal resting heart rate for adults ranges from 60 to 100 beats per minute (bpm). Well-trained athletes may have resting rates as low as 40-60 bpm due to more efficient cardiac function. Factors affecting heart rate include age, fitness level, body temperature, medications, stress, and body position. A consistently high resting heart rate (>100 bpm) is called tachycardia.\",\n",
" \"domain\": \"medicine\",\n",
" \"difficulty\": \"easy\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"MED-006\",\n",
" \"question\": \"How does antibiotic resistance develop?\",\n",
" \"expected_answer\": \"Antibiotic resistance develops through natural selection \u2014 bacteria with random mutations that confer resistance survive antibiotic exposure and reproduce, spreading resistant genes.\",\n",
" \"relevant_context\": \"Antibiotic resistance develops through evolutionary pressure. When bacteria are exposed to antibiotics, those with random genetic mutations conferring resistance survive and reproduce (natural selection). Resistance genes can spread through: vertical transfer (parent to offspring), horizontal gene transfer (conjugation, transformation, transduction between bacteria). Overuse and misuse of antibiotics accelerates this process. WHO considers antibiotic resistance one of the biggest threats to global health.\",\n",
" \"domain\": \"medicine\",\n",
" \"difficulty\": \"medium\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"MED-007\",\n",
" \"question\": \"What is the exact cure rate for stage 4 pancreatic cancer?\",\n",
" \"expected_answer\": \"Stage 4 pancreatic cancer has an extremely poor prognosis with a 5-year survival rate of approximately 1-3%. There is no guaranteed cure.\",\n",
" \"relevant_context\": \"Stage 4 (metastatic) pancreatic cancer has one of the lowest survival rates among cancers. The 5-year survival rate is approximately 1-3%. Median survival after diagnosis is 3-6 months with treatment. Treatments (chemotherapy combinations like FOLFIRINOX, gemcitabine+nab-paclitaxel) can extend survival but rarely achieve complete remission. Clinical trials continue to explore immunotherapy and targeted therapy approaches.\",\n",
" \"domain\": \"medicine\",\n",
" \"difficulty\": \"hard\",\n",
" \"expected_hallucination_type\": \"factual\",\n",
" },\n",
" {\n",
" \"id\": \"MED-008\",\n",
" \"question\": \"What causes Alzheimer's disease?\",\n",
" \"expected_answer\": \"The exact cause of Alzheimer's is not fully understood, but it involves accumulation of amyloid-beta plaques and tau tangles in the brain, along with genetic and environmental factors.\",\n",
" \"relevant_context\": \"Alzheimer's disease is a progressive neurodegenerative disorder. Its exact cause remains unclear, but key pathological features include: accumulation of amyloid-beta protein plaques between neurons, formation of neurofibrillary tangles of tau protein inside neurons, chronic neuroinflammation, and synaptic loss. Risk factors include age, genetics (APOE4 allele), cardiovascular health, and possibly environmental factors. The amyloid hypothesis is debated.\",\n",
" \"domain\": \"medicine\",\n",
" \"difficulty\": \"hard\",\n",
" \"expected_hallucination_type\": \"reasoning\",\n",
" },\n",
" {\n",
" \"id\": \"MED-009\",\n",
" \"question\": \"How long does it take for a broken bone to heal?\",\n",
" \"expected_answer\": \"Most broken bones take 6-8 weeks to heal substantially, though complete healing can take months depending on the bone and severity.\",\n",
" \"relevant_context\": \"Bone fracture healing time varies by location, severity, and patient factors. General timelines: finger/toe bones: 3-5 weeks; wrist/arm: 6-8 weeks; leg bones: 8-12 weeks; femur: 12-16 weeks. Healing occurs in stages: inflammation (days 1-5), soft callus formation (weeks 1-3), hard callus (weeks 3-8), and remodeling (months to years). Factors affecting healing include age, nutrition, blood supply, and whether the fracture is displaced.\",\n",
" \"domain\": \"medicine\",\n",
" \"difficulty\": \"easy\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"MED-010\",\n",
" \"question\": \"Is there a proven link between vaccines and autism?\",\n",
" \"expected_answer\": \"No. Extensive scientific research involving millions of children has found no link between vaccines and autism. The original 1998 study was fraudulent and retracted.\",\n",
" \"relevant_context\": \"There is no scientific evidence linking vaccines to autism. The claim originated from a 1998 study by Andrew Wakefield published in The Lancet, which was later found to be fraudulent \u2014 the paper was retracted in 2010, and Wakefield lost his medical license. Subsequent studies involving millions of children (including a 2019 Danish study of 657,461 children) have consistently found no association between vaccination and autism spectrum disorder.\",\n",
" \"domain\": \"medicine\",\n",
" \"difficulty\": \"medium\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" # === GEOGRAPHY DOMAIN (10 cases) ===\n",
" {\n",
" \"id\": \"GEO-001\",\n",
" \"question\": \"What is the longest river in the world?\",\n",
" \"expected_answer\": \"The Nile River is traditionally considered the longest river at approximately 6,650 km, though some measurements suggest the Amazon may be longer.\",\n",
" \"relevant_context\": \"The title of world's longest river is disputed between the Nile and the Amazon. The Nile, flowing through northeastern Africa, is traditionally measured at approximately 6,650 km (4,130 miles). Recent measurements of the Amazon, including its most distant source in Peru, suggest it may be 6,992 km (4,345 miles). The difference depends on how river length is measured (source definition, channel measurement methods).\",\n",
" \"domain\": \"geography\",\n",
" \"difficulty\": \"medium\",\n",
" \"expected_hallucination_type\": \"contextual\",\n",
" },\n",
" {\n",
" \"id\": \"GEO-002\",\n",
" \"question\": \"What is the capital of Australia?\",\n",
" \"expected_answer\": \"The capital of Australia is Canberra, not Sydney or Melbourne.\",\n",
" \"relevant_context\": \"The capital of Australia is Canberra, located in the Australian Capital Territory (ACT). It became the capital in 1913 as a compromise between rivals Sydney and Melbourne. Canberra was purpose-built as the capital city, designed by architects Walter Burley Griffin and Marion Mahony Griffin. Despite being the capital, it has a much smaller population (~450,000) than Sydney (~5.3 million) or Melbourne (~5.1 million).\",\n",
" \"domain\": \"geography\",\n",
" \"difficulty\": \"easy\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"GEO-003\",\n",
" \"question\": \"How deep is the Mariana Trench?\",\n",
" \"expected_answer\": \"The Mariana Trench reaches approximately 10,994 meters (36,070 feet) at its deepest point, Challenger Deep.\",\n",
" \"relevant_context\": \"The Mariana Trench in the western Pacific Ocean is the deepest known location on Earth. Its deepest point, Challenger Deep, has been measured at approximately 10,994 meters (36,070 feet) below sea level, with measurements varying between surveys (\u00b140 meters). It was first sounded in 1875 by HMS Challenger. Only four crewed descents have reached the bottom: Trieste (1960), Deepsea Challenger (2012), Limiting Factor (2019, multiple dives).\",\n",
" \"domain\": \"geography\",\n",
" \"difficulty\": \"easy\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"GEO-004\",\n",
" \"question\": \"Which countries does the Amazon River flow through?\",\n",
" \"expected_answer\": \"The Amazon River flows through Peru, Colombia, and Brazil, with tributaries in several other South American countries.\",\n",
" \"relevant_context\": \"The Amazon River's main channel flows through three countries: it originates in Peru (Andes Mountains), briefly passes through Colombia, and flows across Brazil to the Atlantic Ocean. Its vast drainage basin (7 million km\u00b2) also covers parts of Bolivia, Venezuela, Ecuador, Guyana, Suriname, and French Guiana. The river discharges approximately 209,000 cubic meters per second \u2014 more than the next seven largest rivers combined.\",\n",
" \"domain\": \"geography\",\n",
" \"difficulty\": \"medium\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"GEO-005\",\n",
" \"question\": \"What is the smallest country in the world?\",\n",
" \"expected_answer\": \"Vatican City is the smallest country in the world by both area (0.44 km\u00b2) and population (~800).\",\n",
" \"relevant_context\": \"Vatican City (officially Vatican City State) is the smallest internationally recognized independent state by both area and population. Located within Rome, Italy, it covers approximately 0.44 km\u00b2 (0.17 sq mi) and has a population of about 800. It became independent from Italy through the 1929 Lateran Treaty. The second smallest country is Monaco (2.02 km\u00b2).\",\n",
" \"domain\": \"geography\",\n",
" \"difficulty\": \"easy\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"GEO-006\",\n",
" \"question\": \"Why is the Dead Sea called 'dead'?\",\n",
" \"expected_answer\": \"The Dead Sea is called 'dead' because its extreme salinity (about 34%) prevents most organisms from living in it.\",\n",
" \"relevant_context\": \"The Dead Sea, bordered by Jordan, Israel, and the West Bank, is called 'dead' because its extreme salinity (~34.2%, nearly 10 times saltier than the ocean) prevents macroscopic aquatic organisms from thriving. However, it's not truly lifeless \u2014 certain halophilic (salt-loving) bacteria and archaea survive there. Its surface is about 430 meters below sea level, making it Earth's lowest land elevation. The sea has been shrinking due to water diversion from the Jordan River.\",\n",
" \"domain\": \"geography\",\n",
" \"difficulty\": \"medium\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"GEO-007\",\n",
" \"question\": \"How many countries are there in Africa?\",\n",
" \"expected_answer\": \"There are 54 recognized sovereign countries in Africa.\",\n",
" \"relevant_context\": \"Africa contains 54 internationally recognized sovereign states (as recognized by the United Nations and the African Union). The most recent addition was South Sudan, which gained independence from Sudan in 2011. The largest country by area is Algeria; the most populous is Nigeria (~220 million). Western Sahara's status remains disputed. Africa is the second-largest and second-most populous continent.\",\n",
" \"domain\": \"geography\",\n",
" \"difficulty\": \"easy\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"GEO-008\",\n",
" \"question\": \"What causes earthquakes?\",\n",
" \"expected_answer\": \"Earthquakes are caused by sudden release of energy in the Earth's crust, usually due to movement along tectonic plate boundaries or fault lines.\",\n",
" \"relevant_context\": \"Earthquakes occur when energy stored in Earth's crust is suddenly released, usually due to movement along geological faults. Tectonic plates constantly move (plate tectonics), and stress accumulates at plate boundaries. When friction is overcome, plates slip suddenly, releasing seismic waves. Most earthquakes occur at plate boundaries: transform (San Andreas), convergent (subduction zones), and divergent boundaries. Intraplate earthquakes can also occur along ancient fault lines.\",\n",
" \"domain\": \"geography\",\n",
" \"difficulty\": \"medium\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"GEO-009\",\n",
" \"question\": \"What is the exact current population of Tokyo?\",\n",
" \"expected_answer\": \"Tokyo's population is approximately 14 million in the city proper, or about 37 million in the greater metropolitan area, making it the world's most populous metropolitan area.\",\n",
" \"relevant_context\": \"Tokyo's population depends on the boundary definition used. Tokyo Metropolis (administrative boundary) has approximately 14 million residents. The Greater Tokyo Area (Tokyo plus surrounding prefectures of Kanagawa, Saitama, and Chiba) has approximately 37-38 million people, making it the world's largest metropolitan area by population. Population figures change continuously and exact numbers depend on the census year and counting methodology.\",\n",
" \"domain\": \"geography\",\n",
" \"difficulty\": \"hard\",\n",
" \"expected_hallucination_type\": \"contextual\",\n",
" },\n",
" {\n",
" \"id\": \"GEO-010\",\n",
" \"question\": \"Is Greenland a continent or an island?\",\n",
" \"expected_answer\": \"Greenland is classified as an island (the world's largest) rather than a continent, primarily due to convention and its geological relationship to the North American plate.\",\n",
" \"relevant_context\": \"Greenland is classified as the world's largest island (approximately 2.166 million km\u00b2) rather than a continent. The distinction is somewhat arbitrary \u2014 Australia (7.692 million km\u00b2) is the smallest continent. The classification depends on tectonic plate structure, geological history, and convention. Greenland sits on the North American tectonic plate and is geologically part of the Canadian Shield. Despite its size, it has only about 56,000 inhabitants, mostly along the coast.\",\n",
" \"domain\": \"geography\",\n",
" \"difficulty\": \"hard\",\n",
" \"expected_hallucination_type\": \"reasoning\",\n",
" },\n",
" # === ADDITIONAL CASES FOR DIVERSITY (5 cases) ===\n",
" {\n",
" \"id\": \"MIX-001\",\n",
" \"question\": \"What is the meaning of life according to Douglas Adams?\",\n",
" \"expected_answer\": \"In Douglas Adams' 'The Hitchhiker's Guide to the Galaxy', the answer to the ultimate question of life, the universe, and everything is 42.\",\n",
" \"relevant_context\": \"In Douglas Adams' science fiction series 'The Hitchhiker's Guide to the Galaxy' (1979), a supercomputer named Deep Thought computes for 7.5 million years and determines that the Answer to the Ultimate Question of Life, the Universe, and Everything is simply the number 42. The joke is that the actual Question was never properly defined.\",\n",
" \"domain\": \"culture\",\n",
" \"difficulty\": \"easy\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"MIX-002\",\n",
" \"question\": \"Can humans survive without sleep?\",\n",
" \"expected_answer\": \"No. Prolonged sleep deprivation causes severe cognitive impairment and can eventually be fatal. The longest documented case of intentional wakefulness was about 11 days.\",\n",
" \"relevant_context\": \"Humans cannot survive indefinitely without sleep. Sleep deprivation causes progressive cognitive decline, hallucinations, immune suppression, and potentially death. The longest scientifically documented period without sleep is approximately 264 hours (11 days) by Randy Gardner in 1964, though he experienced severe cognitive and perceptual disturbances. Fatal familial insomnia (FFI) demonstrates that complete inability to sleep is ultimately lethal.\",\n",
" \"domain\": \"biology\",\n",
" \"difficulty\": \"medium\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"MIX-003\",\n",
" \"question\": \"What is the current world record for the 100m sprint?\",\n",
" \"expected_answer\": \"The world record for the 100m sprint is 9.58 seconds, set by Usain Bolt at the 2009 World Championships in Berlin.\",\n",
" \"relevant_context\": \"The men's 100 meters world record is 9.58 seconds, set by Jamaican sprinter Usain Bolt on August 16, 2009, at the World Championships in Berlin, Germany. He broke his own previous record of 9.69 seconds from the 2008 Beijing Olympics. The women's record is 10.49 seconds by Florence Griffith-Joyner (1988). Bolt also holds the 200m record (19.19s) set at the same 2009 championships.\",\n",
" \"domain\": \"sports\",\n",
" \"difficulty\": \"easy\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"MIX-004\",\n",
" \"question\": \"How do noise-cancelling headphones work?\",\n",
" \"expected_answer\": \"Noise-cancelling headphones use microphones to detect ambient sound and generate an 'anti-noise' signal (inverted phase) that destructively interferes with incoming noise.\",\n",
" \"relevant_context\": \"Active noise-cancelling (ANC) headphones work through destructive interference. External microphones capture ambient sound waves. A processor analyzes these waves and generates an 'anti-noise' signal \u2014 a sound wave with the same amplitude but inverted phase (180\u00b0 out of phase). When the anti-noise meets the original noise, they cancel each other out (destructive interference). This works best for consistent, low-frequency sounds (airplane engines, HVAC hum) and less effectively for irregular, high-frequency sounds (speech).\",\n",
" \"domain\": \"technology\",\n",
" \"difficulty\": \"medium\",\n",
" \"expected_hallucination_type\": \"faithful\",\n",
" },\n",
" {\n",
" \"id\": \"MIX-005\",\n",
" \"question\": \"What would happen if the Earth stopped rotating?\",\n",
" \"expected_answer\": \"If Earth suddenly stopped rotating, catastrophic effects would include extreme winds, massive tsunamis, redistribution of oceans toward the poles, and loss of the magnetic field.\",\n",
" \"relevant_context\": \"If Earth suddenly stopped rotating, the consequences would be catastrophic: (1) Everything not attached to bedrock would continue moving at rotational speed (~1,670 km/h at equator), causing extreme winds and displacement; (2) Oceans would redistribute toward the poles due to loss of centrifugal force, creating a mega-continent at the equator; (3) Day/night cycles would become ~6 months each; (4) The magnetic field (generated by the rotating liquid core) would likely weaken significantly; (5) Climate would become extreme with one side perpetually heated.\",\n",
" \"domain\": \"science\",\n",
" \"difficulty\": \"hard\",\n",
" \"expected_hallucination_type\": \"reasoning\",\n",
" },\n",
" ]\n",
"\n",
" # Validate dataset\n",
" assert len(dataset) >= 50, f\"Dataset has only {len(dataset)} cases (need >= 50)\"\n",
" ids = [d[\"id\"] for d in dataset]\n",
" assert len(ids) == len(set(ids)), \"Duplicate IDs found\"\n",
"\n",
" return dataset\n",
"\n",
"\n",
"# Create and summarize dataset\n",
"eval_dataset = create_synthetic_dataset()\n",
"print(f\"\u2713 Created synthetic dataset with {len(eval_dataset)} test cases\")\n",
"print(f\" Domains: {Counter(d['domain'] for d in eval_dataset)}\")\n",
"print(f\" Difficulties: {Counter(d['difficulty'] for d in eval_dataset)}\")\n",
"print(f\" Hallucination types: {Counter(d['expected_hallucination_type'] for d in eval_dataset)}\")\n",
"print()\n"
],
"outputs": [],
"execution_count": null
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Section 3: Retrieval Strategies (Controlled Experimentation)\n",
"\n",
"Implement and compare 3 retrieval strategies:\n",
"1. **Hybrid:** BM25 + FAISS with Reciprocal Rank Fusion (RRF)\n",
"2. **Dense-only:** FAISS semantic search\n",
"3. **Two-stage:** FAISS \u2192 top-k \u2192 rerank with combined scoring\n",
"\n",
"Each returns top-k documents with similarity scores."
]
},
{
"cell_type": "code",
"metadata": {},
"source": [
"# \u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n",
"# \u2551 SECTION 3: RETRIEVAL STRATEGIES (CONTROLLED EXPERIMENTATION) \u2551\n",
"# \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n",
"\n",
"print(\"=\" * 70)\n",
"print(\"SECTION 3: Building Retrieval Systems\")\n",
"print(\"=\" * 70)\n",
"\n",
"\n",
"class RetrievalSystem:\n",
" \"\"\"\n",
" Manages the knowledge base and provides multiple retrieval strategies.\n",
" \"\"\"\n",
"\n",
" def __init__(self, embedding_model_name: str = \"sentence-transformers/all-MiniLM-L6-v2\"):\n",
" print(f\" Loading embedding model: {embedding_model_name}\")\n",
" self.embedding_model = SentenceTransformer(embedding_model_name)\n",
" self.embedding_dim = 384 # all-MiniLM-L6-v2 dimension\n",
"\n",
" # Storage\n",
" self.documents: List[str] = []\n",
" self.doc_metadata: List[Dict] = []\n",
"\n",
" # Indices\n",
" self.faiss_index: Optional[faiss.IndexFlatIP] = None\n",
" self.bm25: Optional[BM25Okapi] = None\n",
" self.embeddings: Optional[np.ndarray] = None\n",
"\n",
" def build_index(self, documents: List[str], metadata: List[Dict] = None):\n",
" \"\"\"Build both FAISS and BM25 indices from documents.\"\"\"\n",
" self.documents = documents\n",
" self.doc_metadata = metadata or [{\"idx\": i} for i in range(len(documents))]\n",
"\n",
" # Build FAISS index (dense)\n",
" print(\" Building FAISS index...\")\n",
" self.embeddings = self.embedding_model.encode(\n",
" documents,\n",
" batch_size=32,\n",
" normalize_embeddings=True,\n",
" convert_to_numpy=True,\n",
" show_progress_bar=False,\n",
" )\n",
" self.faiss_index = faiss.IndexFlatIP(self.embedding_dim)\n",
" self.faiss_index.add(self.embeddings.astype(\"float32\"))\n",
"\n",
" # Build BM25 index (sparse)\n",
" print(\" Building BM25 index...\")\n",
" tokenized_corpus = [doc.lower().split() for doc in documents]\n",
" self.bm25 = BM25Okapi(tokenized_corpus)\n",
"\n",
" print(f\" \u2713 Indexed {len(documents)} documents\")\n",
"\n",
" def retrieve_hybrid(self, query: str, k: int = 5, rrf_k: int = 60) -> List[Dict]:\n",
" \"\"\"\n",
" Strategy 1: Hybrid BM25 + FAISS with Reciprocal Rank Fusion.\n",
" Combines sparse (lexical) and dense (semantic) retrieval.\n",
" \"\"\"\n",
" # Dense retrieval\n",
" q_emb = self.embedding_model.encode(\n",
" [query], normalize_embeddings=True, convert_to_numpy=True\n",
" ).astype(\"float32\")\n",
" dense_scores, dense_indices = self.faiss_index.search(q_emb, k)\n",
" dense_ranking = dense_indices[0].tolist()\n",
" dense_score_map = {idx: float(score) for idx, score in zip(dense_indices[0], dense_scores[0])}\n",
"\n",
" # Sparse retrieval\n",
" bm25_scores = self.bm25.get_scores(query.lower().split())\n",
" sparse_ranking = np.argsort(bm25_scores)[::-1][:k].tolist()\n",
"\n",
" # Reciprocal Rank Fusion\n",
" fused_scores = {}\n",
" for rank, idx in enumerate(dense_ranking):\n",
" if idx >= 0: # FAISS can return -1 for empty results\n",
" fused_scores[idx] = fused_scores.get(idx, 0) + 1.0 / (rrf_k + rank + 1)\n",
" for rank, idx in enumerate(sparse_ranking):\n",
" fused_scores[idx] = fused_scores.get(idx, 0) + 1.0 / (rrf_k + rank + 1)\n",
"\n",
" # Sort by fused score\n",
" ranked = sorted(fused_scores.items(), key=lambda x: x[1], reverse=True)[:k]\n",
"\n",
" results = []\n",
" for idx, fused_score in ranked:\n",
" results.append({\n",
" \"document\": self.documents[idx],\n",
" \"index\": idx,\n",
" \"score\": fused_score,\n",
" \"dense_score\": dense_score_map.get(idx, 0.0),\n",
" \"strategy\": \"hybrid\",\n",
" })\n",
" return results\n",
"\n",
" def retrieve_dense(self, query: str, k: int = 5) -> List[Dict]:\n",
" \"\"\"\n",
" Strategy 2: Dense-only retrieval using FAISS (semantic search).\n",
" \"\"\"\n",
" q_emb = self.embedding_model.encode(\n",
" [query], normalize_embeddings=True, convert_to_numpy=True\n",
" ).astype(\"float32\")\n",
" scores, indices = self.faiss_index.search(q_emb, k)\n",
"\n",
" results = []\n",
" for idx, score in zip(indices[0], scores[0]):\n",
" if idx >= 0:\n",
" results.append({\n",
" \"document\": self.documents[idx],\n",
" \"index\": int(idx),\n",
" \"score\": float(score),\n",
" \"dense_score\": float(score),\n",
" \"strategy\": \"dense\",\n",
" })\n",
" return results\n",
"\n",
" def retrieve_two_stage(self, query: str, k: int = 5, first_stage_k: int = 20) -> List[Dict]:\n",
" \"\"\"\n",
" Strategy 3: Two-stage retrieval.\n",
" Stage 1: FAISS retrieves top-k_first candidates.\n",
" Stage 2: Re-rank candidates using cross-encoder-style similarity scoring.\n",
" \"\"\"\n",
" # Stage 1: Broad FAISS retrieval\n",
" q_emb = self.embedding_model.encode(\n",
" [query], normalize_embeddings=True, convert_to_numpy=True\n",
" ).astype(\"float32\")\n",
" scores, indices = self.faiss_index.search(q_emb, first_stage_k)\n",
"\n",
" candidates = []\n",
" for idx, score in zip(indices[0], scores[0]):\n",
" if idx >= 0:\n",
" candidates.append((int(idx), float(score)))\n",
"\n",
" # Stage 2: Re-rank using token overlap + embedding similarity (lightweight cross-encoder proxy)\n",
" query_tokens = set(query.lower().split())\n",
" reranked = []\n",
" for idx, dense_score in candidates:\n",
" doc = self.documents[idx]\n",
" doc_tokens = set(doc.lower().split())\n",
"\n",
" # Token overlap (Jaccard-like)\n",
" overlap = len(query_tokens & doc_tokens) / max(len(query_tokens | doc_tokens), 1)\n",
"\n",
" # Combined score: weighted sum of dense similarity and token overlap\n",
" combined_score = 0.7 * dense_score + 0.3 * overlap\n",
" reranked.append((idx, combined_score, dense_score))\n",
"\n",
" # Sort by combined score\n",
" reranked.sort(key=lambda x: x[1], reverse=True)\n",
" results = []\n",
" for idx, combined_score, dense_score in reranked[:k]:\n",
" results.append({\n",
" \"document\": self.documents[idx],\n",
" \"index\": idx,\n",
" \"score\": combined_score,\n",
" \"dense_score\": dense_score,\n",
" \"strategy\": \"two_stage\",\n",
" })\n",
" return results\n",
"\n",
" def add_document(self, text: str, metadata: Dict = None):\n",
" \"\"\"\n",
" Add a new document to the knowledge base (dynamic update).\n",
" Updates both FAISS and BM25 indices.\n",
" \"\"\"\n",
" self.documents.append(text)\n",
" self.doc_metadata.append(metadata or {\"idx\": len(self.documents) - 1})\n",
"\n",
" # Update FAISS\n",
" new_embedding = self.embedding_model.encode(\n",
" [text], normalize_embeddings=True, convert_to_numpy=True\n",
" ).astype(\"float32\")\n",
" self.faiss_index.add(new_embedding)\n",
" self.embeddings = np.vstack([self.embeddings, new_embedding])\n",
"\n",
" # Rebuild BM25 (BM25Okapi doesn't support incremental updates)\n",
" tokenized_corpus = [doc.lower().split() for doc in self.documents]\n",
" self.bm25 = BM25Okapi(tokenized_corpus)\n",
"\n",
"\n",
"# Build knowledge base from dataset contexts\n",
"print(\" Initializing retrieval system...\")\n",
"retrieval_system = RetrievalSystem()\n",
"\n",
"# Extract all contexts as our knowledge base\n",
"knowledge_base = [item[\"relevant_context\"] for item in eval_dataset]\n",
"kb_metadata = [{\"id\": item[\"id\"], \"domain\": item[\"domain\"]} for item in eval_dataset]\n",
"retrieval_system.build_index(knowledge_base, kb_metadata)\n",
"print()\n"
],
"outputs": [],
"execution_count": null
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Section 4 & 5: RAG Pipeline + Models\n",
"\n",
"**Pipeline:** query \u2192 retrieval \u2192 generation \u2192 verification\n",
"\n",
"**Models:**\n",
"1. **Primary:** TinyLlama-1.1B-Chat (lightweight open-source, chat-capable)\n",
"2. **Secondary:** DistilGPT2 (very small, fast baseline)\n",
"3. **Fallback:** Rule-based extractive generator (always available)\n",
"\n",
"Rules: Answer ONLY from retrieved context. If insufficient context \u2192 \"I don't know\""
]
},
{
"cell_type": "code",
"metadata": {},
"source": [
"# \u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n",
"# \u2551 SECTION 4 & 5: RAG PIPELINE + MODELS \u2551\n",
"# \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n",
"\n",
"print(\"=\" * 70)\n",
"print(\"SECTION 4 & 5: RAG Pipeline + Model Loading\")\n",
"print(\"=\" * 70)\n",
"\n",
"\n",
"class GenerationModel:\n",
" \"\"\"Wrapper for text generation models.\"\"\"\n",
"\n",
" def __init__(self, model_name: str, model_type: str = \"pipeline\"):\n",
" self.model_name = model_name\n",
" self.model_type = model_type\n",
" self.pipe = None\n",
" self.loaded = False\n",
"\n",
" def load(self):\n",
" \"\"\"Load the model. Handles failures gracefully.\"\"\"\n",
" try:\n",
" print(f\" Loading model: {self.model_name}...\")\n",
" device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
" dtype = torch.bfloat16 if torch.cuda.is_available() else torch.float32\n",
"\n",
" self.pipe = pipeline(\n",
" \"text-generation\",\n",
" model=self.model_name,\n",
" torch_dtype=dtype,\n",
" device_map=\"auto\" if torch.cuda.is_available() else None,\n",
" device=None if torch.cuda.is_available() else device,\n",
" )\n",
" self.loaded = True\n",
" print(f\" \u2713 Model loaded: {self.model_name}\")\n",
" except Exception as e:\n",
" print(f\" \u2717 Failed to load {self.model_name}: {e}\")\n",
" self.loaded = False\n",
"\n",
" def generate(self, prompt: str, max_new_tokens: int = 256) -> str:\n",
" \"\"\"Generate text from prompt.\"\"\"\n",
" if not self.loaded:\n",
" return \"[MODEL NOT LOADED]\"\n",
" try:\n",
" outputs = self.pipe(\n",
" prompt,\n",
" max_new_tokens=max_new_tokens,\n",
" do_sample=False,\n",
" repetition_penalty=1.1,\n",
" return_full_text=False,\n",
" pad_token_id=self.pipe.tokenizer.eos_token_id,\n",
" )\n",
" return outputs[0][\"generated_text\"].strip()\n",
" except Exception as e:\n",
" return f\"[GENERATION ERROR: {e}]\"\n",
"\n",
"\n",
"class SimpleGeneratorFallback:\n",
" \"\"\"\n",
" Fallback generator that uses extractive summarization from context.\n",
" Does not require GPU \u2014 purely rule-based.\n",
" \"\"\"\n",
"\n",
" def __init__(self):\n",
" self.model_name = \"extractive-fallback\"\n",
" self.loaded = True\n",
"\n",
" def load(self):\n",
" pass\n",
"\n",
" def generate(self, prompt: str, max_new_tokens: int = 256) -> str:\n",
" \"\"\"Extract most relevant sentence from context as answer.\"\"\"\n",
" # Parse context from prompt\n",
" context_start = prompt.find(\"Context:\")\n",
" question_start = prompt.find(\"Question:\")\n",
"\n",
" if context_start == -1 or question_start == -1:\n",
" return \"I don't know based on the provided context.\"\n",
"\n",
" context = prompt[context_start + 8:question_start].strip()\n",
" question = prompt[question_start + 9:].strip()\n",
"\n",
" # Simple extractive approach: find sentence with most keyword overlap\n",
" sentences = [s.strip() for s in context.replace(\"\\n\", \" \").split(\".\") if len(s.strip()) > 10]\n",
" if not sentences:\n",
" return \"I don't know based on the provided context.\"\n",
"\n",
" question_words = set(question.lower().split())\n",
" best_sentence = \"\"\n",
" best_score = 0\n",
"\n",
" for sent in sentences:\n",
" sent_words = set(sent.lower().split())\n",
" overlap = len(question_words & sent_words)\n",
" if overlap > best_score:\n",
" best_score = overlap\n",
" best_sentence = sent\n",
"\n",
" if best_score == 0:\n",
" return \"I don't know based on the provided context.\"\n",
"\n",
" return best_sentence.strip() + \".\"\n",
"\n",
"\n",
"# Initialize models\n",
"print()\n",
"\n",
"# Model 1: TinyLlama (lightweight, open-source, chat-capable)\n",
"model_primary = GenerationModel(\"TinyLlama/TinyLlama-1.1B-Chat-v1.0\", model_type=\"pipeline\")\n",
"model_primary.load()\n",
"\n",
"# Model 2: DistilGPT2 (very small, fast, good for comparison)\n",
"model_secondary = GenerationModel(\"distilgpt2\", model_type=\"pipeline\")\n",
"model_secondary.load()\n",
"\n",
"# Model 3: Extractive fallback (rule-based, always available)\n",
"model_fallback = SimpleGeneratorFallback()\n",
"\n",
"# Collect available models\n",
"available_models = []\n",
"if model_primary.loaded:\n",
" available_models.append((\"TinyLlama-1.1B-Chat\", model_primary))\n",
"if model_secondary.loaded:\n",
" available_models.append((\"DistilGPT2\", model_secondary))\n",
"available_models.append((\"Extractive-Fallback\", model_fallback))\n",
"\n",
"print(f\"\\n \u2713 {len(available_models)} models available: {[m[0] for m in available_models]}\")\n",
"print()\n",
"\n",
"\n",
"def build_rag_prompt(question: str, contexts: List[str]) -> str:\n",
" \"\"\"Build a RAG prompt with retrieved context.\"\"\"\n",
" context_text = \"\\n\\n\".join(contexts)\n",
" prompt = f\"\"\"Answer the question using ONLY the context provided below. If the context does not contain enough information to answer the question, respond with \"I don't know based on the provided context.\"\n",
"\n",
"Context:\n",
"{context_text}\n",
"\n",
"Question: {question}\n",
"\n",
"Answer:\"\"\"\n",
" return prompt\n",
"\n",
"\n",
"def build_chat_prompt(question: str, contexts: List[str], tokenizer=None) -> str:\n",
" \"\"\"Build a chat-formatted prompt for chat models.\"\"\"\n",
" context_text = \"\\n\\n\".join(contexts)\n",
" messages = [\n",
" {\n",
" \"role\": \"system\",\n",
" \"content\": \"You are a helpful assistant. Answer questions using ONLY the provided context. If the context is insufficient, say 'I don't know based on the provided context.' Be concise and factual.\",\n",
" },\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": f\"Context:\\n{context_text}\\n\\nQuestion: {question}\",\n",
" },\n",
" ]\n",
" if tokenizer and hasattr(tokenizer, \"apply_chat_template\"):\n",
" try:\n",
" return tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)\n",
" except Exception:\n",
" pass\n",
" return build_rag_prompt(question, contexts)\n"
],
"outputs": [],
"execution_count": null
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Section 6: Agent System (Lightweight)\n",
"\n",
"4 agents implemented as simple classes (no external frameworks):\n",
"\n",
"| Agent | Role |\n",
"|-------|------|\n",
"| **PlannerAgent** | Rule-based step planning (retrieve \u2192 generate \u2192 verify) |\n",
"| **ExecutorAgent** | Executes retrieval and generation, calls tools |\n",
"| **CriticAgent** | Evaluates grounding, assigns hallucination type + confidence |\n",
"| **LoggingAgent** | Records all system activity with structured prefixes |"
]
},
{
"cell_type": "code",
"metadata": {},
"source": [
"# \u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n",
"# \u2551 SECTION 6: AGENT SYSTEM (LIGHTWEIGHT) \u2551\n",
"# \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n",
"\n",
"print(\"=\" * 70)\n",
"print(\"SECTION 6: Agent System\")\n",
"print(\"=\" * 70)\n",
"\n",
"\n",
"class LoggingAgent:\n",
" \"\"\"\n",
" Records all system activity for analysis.\n",
" Provides structured logging with prefixed messages.\n",
" \"\"\"\n",
"\n",
" def __init__(self):\n",
" self.logs: List[Dict] = []\n",
"\n",
" def log(self, agent: str, action: str, details: Dict = None):\n",
" \"\"\"Log an agent action with timestamp.\"\"\"\n",
" entry = {\n",
" \"timestamp\": time.time(),\n",
" \"agent\": agent,\n",
" \"action\": action,\n",
" \"details\": details or {},\n",
" }\n",
" self.logs.append(entry)\n",
" prefix = f\"[{agent.upper()}]\"\n",
" print(f\" {prefix:12s} {action}\")\n",
"\n",
" def get_logs(self) -> List[Dict]:\n",
" \"\"\"Return all logs.\"\"\"\n",
" return self.logs\n",
"\n",
" def get_summary(self) -> Dict:\n",
" \"\"\"Return summary statistics.\"\"\"\n",
" return {\n",
" \"total_entries\": len(self.logs),\n",
" \"by_agent\": Counter(log[\"agent\"] for log in self.logs),\n",
" \"by_action\": Counter(log[\"action\"] for log in self.logs),\n",
" }\n",
"\n",
" def clear(self):\n",
" \"\"\"Clear all logs.\"\"\"\n",
" self.logs = []\n",
"\n",
"\n",
"class PlannerAgent:\n",
" \"\"\"\n",
" Rule-based planner that decides the execution steps.\n",
" Analyzes the query and determines the optimal workflow.\n",
" \"\"\"\n",
"\n",
" def __init__(self, logger: LoggingAgent):\n",
" self.logger = logger\n",
"\n",
" def plan(self, query: str, retrieval_strategy: str = \"hybrid\") -> List[Dict]:\n",
" \"\"\"\n",
" Create an execution plan based on the query.\n",
" Returns ordered list of steps.\n",
" \"\"\"\n",
" self.logger.log(\"PLANNER\", \"Planning started\", {\"query\": query[:80]})\n",
"\n",
" steps = [\n",
" {\"step\": 1, \"action\": \"retrieve\", \"strategy\": retrieval_strategy, \"k\": 5},\n",
" {\"step\": 2, \"action\": \"generate\", \"model\": \"primary\"},\n",
" {\"step\": 3, \"action\": \"verify\", \"checks\": [\"grounding\", \"confidence\"]},\n",
" ]\n",
"\n",
" # Adaptive planning based on query complexity\n",
" query_words = query.split()\n",
" if len(query_words) > 15 or \"?\" in query and query.count(\"?\") > 1:\n",
" # Complex query: add extra retrieval\n",
" steps[0][\"k\"] = 7\n",
" self.logger.log(\"PLANNER\", \"Complex query detected \u2014 increased k to 7\")\n",
"\n",
" if any(word in query.lower() for word in [\"exact\", \"precise\", \"specific\", \"how many\"]):\n",
" # Precision-required query: add stricter verification\n",
" steps[2][\"checks\"].append(\"precision_check\")\n",
" self.logger.log(\"PLANNER\", \"Precision query \u2014 added strict verification\")\n",
"\n",
" self.logger.log(\"PLANNER\", f\"Plan created: {len(steps)} steps\", {\"steps\": steps})\n",
" return steps\n",
"\n",
"\n",
"class ExecutorAgent:\n",
" \"\"\"\n",
" Executes retrieval and generation steps from the plan.\n",
" Calls tools and manages the pipeline execution.\n",
" \"\"\"\n",
"\n",
" def __init__(self, retrieval_system: RetrievalSystem, models: List[tuple], logger: LoggingAgent):\n",
" self.retrieval_system = retrieval_system\n",
" self.models = {name: model for name, model in models}\n",
" self.logger = logger\n",
"\n",
" def execute(self, plan: List[Dict], query: str) -> Dict:\n",
" \"\"\"Execute the plan and return results.\"\"\"\n",
" result = {\"query\": query, \"steps_executed\": []}\n",
"\n",
" for step in plan:\n",
" action = step[\"action\"]\n",
" start_time = time.time()\n",
"\n",
" if action == \"retrieve\":\n",
" retrieved = self._execute_retrieval(query, step[\"strategy\"], step.get(\"k\", 5))\n",
" result[\"retrieved_docs\"] = retrieved\n",
" result[\"retrieval_strategy\"] = step[\"strategy\"]\n",
"\n",
" elif action == \"generate\":\n",
" contexts = [doc[\"document\"] for doc in result.get(\"retrieved_docs\", [])]\n",
" answer = self._execute_generation(query, contexts)\n",
" result[\"generated_answer\"] = answer\n",
"\n",
" elif action == \"verify\":\n",
" # Verification handled by CriticAgent\n",
" pass\n",
"\n",
" elapsed = time.time() - start_time\n",
" result[\"steps_executed\"].append({\"action\": action, \"latency\": elapsed})\n",
" self.logger.log(\"EXECUTOR\", f\"Step '{action}' completed\", {\"latency_ms\": round(elapsed * 1000, 1)})\n",
"\n",
" return result\n",
"\n",
" def _execute_retrieval(self, query: str, strategy: str, k: int) -> List[Dict]:\n",
" \"\"\"Execute retrieval using specified strategy.\"\"\"\n",
" self.logger.log(\"EXECUTOR\", f\"Retrieving with strategy='{strategy}', k={k}\")\n",
"\n",
" if strategy == \"hybrid\":\n",
" return self.retrieval_system.retrieve_hybrid(query, k=k)\n",
" elif strategy == \"dense\":\n",
" return self.retrieval_system.retrieve_dense(query, k=k)\n",
" elif strategy == \"two_stage\":\n",
" return self.retrieval_system.retrieve_two_stage(query, k=k)\n",
" else:\n",
" self.logger.log(\"EXECUTOR\", f\"Unknown strategy '{strategy}', falling back to hybrid\")\n",
" return self.retrieval_system.retrieve_hybrid(query, k=k)\n",
"\n",
" def _execute_generation(self, query: str, contexts: List[str]) -> str:\n",
" \"\"\"Generate answer using the best available model.\"\"\"\n",
" if not contexts:\n",
" return \"I don't know based on the provided context.\"\n",
"\n",
" # Use primary model with chat template if available\n",
" model_name, model = available_models[0]\n",
"\n",
" if hasattr(model, \"pipe\") and model.pipe and hasattr(model.pipe, \"tokenizer\"):\n",
" prompt = build_chat_prompt(query, contexts, model.pipe.tokenizer)\n",
" else:\n",
" prompt = build_rag_prompt(query, contexts)\n",
"\n",
" answer = model.generate(prompt, max_new_tokens=200)\n",
" self.logger.log(\"EXECUTOR\", f\"Generated answer using {model_name}\", {\"answer_length\": len(answer)})\n",
" return answer\n",
"\n",
"\n",
"class CriticAgent:\n",
" \"\"\"\n",
" Evaluates answer quality, grounding, and hallucination.\n",
" Assigns hallucination type and confidence score.\n",
" \"\"\"\n",
"\n",
" def __init__(self, logger: LoggingAgent):\n",
" self.logger = logger\n",
"\n",
" def evaluate(self, query: str, answer: str, contexts: List[str]) -> Dict:\n",
" \"\"\"\n",
" Evaluate the generated answer against retrieved context.\n",
" Returns grounding assessment, hallucination classification, and confidence.\n",
" \"\"\"\n",
" self.logger.log(\"CRITIC\", \"Evaluating answer quality\")\n",
"\n",
" # Compute context overlap\n",
" overlap_score = self._compute_context_overlap(answer, contexts)\n",
"\n",
" # Grounding check\n",
" grounding = self._check_grounding(answer, contexts)\n",
"\n",
" # Hallucination classification\n",
" hallucination_type = self._classify_hallucination(answer, contexts, overlap_score)\n",
"\n",
" # Confidence score (heuristic)\n",
" confidence = self._compute_confidence(overlap_score, grounding, answer)\n",
"\n",
" evaluation = {\n",
" \"context_overlap_score\": overlap_score,\n",
" \"grounding\": grounding,\n",
" \"hallucination_type\": hallucination_type,\n",
" \"confidence_score\": confidence,\n",
" }\n",
"\n",
" self.logger.log(\"CRITIC\", f\"Evaluation complete\", {\n",
" \"overlap\": round(overlap_score, 3),\n",
" \"confidence\": round(confidence, 3),\n",
" \"hallucination\": hallucination_type,\n",
" })\n",
"\n",
" return evaluation\n",
"\n",
" def _compute_context_overlap(self, answer: str, contexts: List[str]) -> float:\n",
" \"\"\"Compute token-level overlap between answer and contexts.\"\"\"\n",
" if not answer or not contexts:\n",
" return 0.0\n",
"\n",
" answer_tokens = set(answer.lower().split())\n",
" context_tokens = set()\n",
" for ctx in contexts:\n",
" context_tokens.update(ctx.lower().split())\n",
"\n",
" # Remove common stopwords for meaningful overlap\n",
" stopwords = {\"the\", \"a\", \"an\", \"is\", \"are\", \"was\", \"were\", \"in\", \"on\", \"at\", \"to\",\n",
" \"for\", \"of\", \"and\", \"or\", \"it\", \"its\", \"this\", \"that\", \"with\", \"by\",\n",
" \"from\", \"as\", \"be\", \"has\", \"had\", \"have\", \"not\", \"but\", \"if\", \"they\"}\n",
"\n",
" answer_meaningful = answer_tokens - stopwords\n",
" context_meaningful = context_tokens - stopwords\n",
"\n",
" if not answer_meaningful:\n",
" return 0.0\n",
"\n",
" overlap = len(answer_meaningful & context_meaningful) / len(answer_meaningful)\n",
" return overlap\n",
"\n",
" def _check_grounding(self, answer: str, contexts: List[str]) -> Dict:\n",
" \"\"\"Check if key claims in the answer are grounded in context.\"\"\"\n",
" if not answer or not contexts:\n",
" return {\"grounded\": False, \"coverage\": 0.0}\n",
"\n",
" combined_context = \" \".join(contexts).lower()\n",
" answer_sentences = [s.strip() for s in answer.split(\".\") if len(s.strip()) > 5]\n",
"\n",
" grounded_count = 0\n",
" for sent in answer_sentences:\n",
" # Check if key words from sentence appear in context\n",
" sent_words = set(sent.lower().split()) - {\"the\", \"a\", \"an\", \"is\", \"are\", \"was\", \"it\"}\n",
" if len(sent_words) == 0:\n",
" continue\n",
" context_words = set(combined_context.split())\n",
" match_ratio = len(sent_words & context_words) / len(sent_words)\n",
" if match_ratio > 0.5:\n",
" grounded_count += 1\n",
"\n",
" coverage = grounded_count / max(len(answer_sentences), 1)\n",
" return {\"grounded\": coverage > 0.6, \"coverage\": coverage}\n",
"\n",
" def _classify_hallucination(self, answer: str, contexts: List[str], overlap: float) -> str:\n",
" \"\"\"\n",
" Classify the type of potential hallucination.\n",
" Categories: factual, contextual, reasoning, faithful\n",
" \"\"\"\n",
" if overlap > 0.7:\n",
" return \"faithful\"\n",
" elif overlap > 0.5:\n",
" # Partially grounded \u2014 check if it's contextual or reasoning\n",
" if any(word in answer.lower() for word in [\"because\", \"therefore\", \"thus\", \"since\", \"implies\"]):\n",
" return \"reasoning\"\n",
" else:\n",
" return \"contextual\"\n",
" elif overlap > 0.3:\n",
" return \"contextual\"\n",
" else:\n",
" return \"factual\"\n",
"\n",
" def _compute_confidence(self, overlap: float, grounding: Dict, answer: str) -> float:\n",
" \"\"\"\n",
" Compute a heuristic confidence score [0, 1].\n",
" Higher = more likely to be correct and grounded.\n",
" \"\"\"\n",
" # Base confidence from overlap\n",
" conf = overlap * 0.5\n",
"\n",
" # Bonus for grounding\n",
" if grounding.get(\"grounded\", False):\n",
" conf += 0.3\n",
" else:\n",
" conf += grounding.get(\"coverage\", 0) * 0.2\n",
"\n",
" # Penalty for \"I don't know\" (but it's actually good behavior)\n",
" if \"i don't know\" in answer.lower() or \"don't know\" in answer.lower():\n",
" conf = 0.8 # High confidence in abstention\n",
"\n",
" # Penalty for very long answers (more likely to hallucinate)\n",
" if len(answer.split()) > 100:\n",
" conf *= 0.9\n",
"\n",
" # Penalty for very short answers\n",
" if len(answer.split()) < 3:\n",
" conf *= 0.7\n",
"\n",
" return min(max(conf, 0.0), 1.0)\n",
"\n",
"\n",
"# Initialize agents\n",
"logger_agent = LoggingAgent()\n",
"planner_agent = PlannerAgent(logger_agent)\n",
"executor_agent = ExecutorAgent(retrieval_system, available_models, logger_agent)\n",
"critic_agent = CriticAgent(logger_agent)\n",
"\n",
"print(\" \u2713 Agent system initialized:\")\n",
"print(\" - PlannerAgent (rule-based step planning)\")\n",
"print(\" - ExecutorAgent (retrieval + generation)\")\n",
"print(\" - CriticAgent (grounding + hallucination detection)\")\n",
"print(\" - LoggingAgent (structured activity recording)\")\n",
"print()\n"
],
"outputs": [],
"execution_count": null
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Section 7: Tools\n",
"\n",
"Callable tool classes:\n",
"1. **RetrievalTool** \u2014 multi-strategy document retrieval\n",
"2. **VerificationTool** \u2014 answer grounding verification\n",
"3. **KeywordSearchTool** \u2014 simple keyword lookup in knowledge base"
]
},
{
"cell_type": "code",
"metadata": {},
"source": [
"# \u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n",
"# \u2551 SECTION 7: TOOLS \u2551\n",
"# \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n",
"\n",
"print(\"=\" * 70)\n",
"print(\"SECTION 7: Tools\")\n",
"print(\"=\" * 70)\n",
"\n",
"\n",
"class RetrievalTool:\n",
" \"\"\"Tool for document retrieval with configurable strategy.\"\"\"\n",
"\n",
" name = \"retrieval_tool\"\n",
" description = \"Retrieves relevant documents from the knowledge base.\"\n",
"\n",
" def __init__(self, retrieval_system: RetrievalSystem):\n",
" self.retrieval_system = retrieval_system\n",
"\n",
" def __call__(self, query: str, strategy: str = \"hybrid\", k: int = 5) -> List[Dict]:\n",
" \"\"\"Execute retrieval.\"\"\"\n",
" if strategy == \"hybrid\":\n",
" return self.retrieval_system.retrieve_hybrid(query, k)\n",
" elif strategy == \"dense\":\n",
" return self.retrieval_system.retrieve_dense(query, k)\n",
" elif strategy == \"two_stage\":\n",
" return self.retrieval_system.retrieve_two_stage(query, k)\n",
" return self.retrieval_system.retrieve_hybrid(query, k)\n",
"\n",
"\n",
"class VerificationTool:\n",
" \"\"\"Tool for verifying answer grounding against context.\"\"\"\n",
"\n",
" name = \"verification_tool\"\n",
" description = \"Verifies if an answer is grounded in the provided context.\"\n",
"\n",
" def __call__(self, answer: str, contexts: List[str]) -> Dict:\n",
" \"\"\"Verify answer grounding.\"\"\"\n",
" if not answer or not contexts:\n",
" return {\"verified\": False, \"reason\": \"Empty answer or context\"}\n",
"\n",
" # Token overlap check\n",
" answer_tokens = set(answer.lower().split())\n",
" context_tokens = set()\n",
" for ctx in contexts:\n",
" context_tokens.update(ctx.lower().split())\n",
"\n",
" stopwords = {\"the\", \"a\", \"an\", \"is\", \"are\", \"was\", \"were\", \"in\", \"on\", \"at\",\n",
" \"to\", \"for\", \"of\", \"and\", \"or\", \"it\", \"this\", \"that\", \"with\"}\n",
" answer_meaningful = answer_tokens - stopwords\n",
" context_meaningful = context_tokens - stopwords\n",
"\n",
" if not answer_meaningful:\n",
" return {\"verified\": False, \"reason\": \"No meaningful tokens in answer\"}\n",
"\n",
" overlap = len(answer_meaningful & context_meaningful) / len(answer_meaningful)\n",
" unsupported = answer_meaningful - context_meaningful\n",
"\n",
" return {\n",
" \"verified\": overlap > 0.5,\n",
" \"overlap_score\": round(overlap, 4),\n",
" \"unsupported_tokens\": list(unsupported)[:10],\n",
" \"reason\": \"Sufficient grounding\" if overlap > 0.5 else \"Insufficient grounding\",\n",
" }\n",
"\n",
"\n",
"class KeywordSearchTool:\n",
" \"\"\"Simple keyword search tool for quick lookups.\"\"\"\n",
"\n",
" name = \"keyword_search_tool\"\n",
" description = \"Searches for specific keywords in the knowledge base.\"\n",
"\n",
" def __init__(self, documents: List[str]):\n",
" self.documents = documents\n",
"\n",
" def __call__(self, keyword: str) -> List[Dict]:\n",
" \"\"\"Find documents containing the keyword.\"\"\"\n",
" results = []\n",
" keyword_lower = keyword.lower()\n",
" for i, doc in enumerate(self.documents):\n",
" if keyword_lower in doc.lower():\n",
" results.append({\n",
" \"index\": i,\n",
" \"document\": doc,\n",
" \"keyword_count\": doc.lower().count(keyword_lower),\n",
" })\n",
" return sorted(results, key=lambda x: x[\"keyword_count\"], reverse=True)[:5]\n",
"\n",
"\n",
"# Initialize tools\n",
"retrieval_tool = RetrievalTool(retrieval_system)\n",
"verification_tool = VerificationTool()\n",
"keyword_tool = KeywordSearchTool(knowledge_base)\n",
"\n",
"print(\" \u2713 Tools initialized:\")\n",
"print(\" - RetrievalTool (multi-strategy document retrieval)\")\n",
"print(\" - VerificationTool (answer grounding verification)\")\n",
"print(\" - KeywordSearchTool (simple keyword lookup)\")\n",
"print()\n"
],
"outputs": [],
"execution_count": null
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Section 8: Hallucination Detection\n",
"\n",
"Multi-signal hallucination detector combining:\n",
"- Context overlap score (token-level)\n",
"- Grounding check (sentence-level)\n",
"- Semantic similarity (embedding-based)\n",
"- Factual consistency indicators (numbers, entities)\n",
"\n",
"**Classifications:** `faithful`, `factual`, `contextual`, `reasoning`\n",
"\n",
"Also computes a heuristic **confidence score** \u2208 [0, 1]."
]
},
{
"cell_type": "code",
"metadata": {},
"source": [
"# \u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n",
"# \u2551 SECTION 8: HALLUCINATION DETECTION \u2551\n",
"# \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n",
"\n",
"print(\"=\" * 70)\n",
"print(\"SECTION 8: Hallucination Detection\")\n",
"print(\"=\" * 70)\n",
"\n",
"\n",
"class HallucinationDetector:\n",
" \"\"\"\n",
" Multi-signal hallucination detector.\n",
" Combines context overlap, grounding checks, and optional LLM-as-judge.\n",
" \"\"\"\n",
"\n",
" def __init__(self, embedding_model: SentenceTransformer = None):\n",
" self.embedding_model = embedding_model\n",
"\n",
" def detect(self, answer: str, contexts: List[str], query: str = \"\") -> Dict:\n",
" \"\"\"\n",
" Run hallucination detection pipeline.\n",
" Returns classification, confidence, and detailed signals.\n",
" \"\"\"\n",
" signals = {}\n",
"\n",
" # Signal 1: Context overlap score\n",
" signals[\"context_overlap\"] = self._context_overlap(answer, contexts)\n",
"\n",
" # Signal 2: Grounding check (sentence-level)\n",
" signals[\"grounding\"] = self._grounding_check(answer, contexts)\n",
"\n",
" # Signal 3: Semantic similarity (if embedding model available)\n",
" if self.embedding_model:\n",
" signals[\"semantic_similarity\"] = self._semantic_similarity(answer, contexts)\n",
"\n",
" # Signal 4: Factual consistency indicators\n",
" signals[\"factual_indicators\"] = self._factual_indicators(answer, contexts)\n",
"\n",
" # Aggregate signals into classification\n",
" classification = self._classify(signals)\n",
"\n",
" return {\n",
" \"classification\": classification[\"type\"],\n",
" \"confidence\": classification[\"confidence\"],\n",
" \"signals\": signals,\n",
" \"is_hallucination\": classification[\"type\"] != \"faithful\",\n",
" }\n",
"\n",
" def _context_overlap(self, answer: str, contexts: List[str]) -> float:\n",
" \"\"\"Token-level overlap between answer and context.\"\"\"\n",
" if not answer or not contexts:\n",
" return 0.0\n",
"\n",
" answer_tokens = set(answer.lower().split())\n",
" context_tokens = set()\n",
" for ctx in contexts:\n",
" context_tokens.update(ctx.lower().split())\n",
"\n",
" stopwords = {\"the\", \"a\", \"an\", \"is\", \"are\", \"was\", \"were\", \"in\", \"on\", \"at\",\n",
" \"to\", \"for\", \"of\", \"and\", \"or\", \"it\", \"its\", \"this\", \"that\",\n",
" \"with\", \"by\", \"from\", \"as\", \"be\", \"has\", \"had\", \"have\"}\n",
" answer_meaningful = answer_tokens - stopwords\n",
" context_meaningful = context_tokens - stopwords\n",
"\n",
" if not answer_meaningful:\n",
" return 0.5 # Neutral if no meaningful tokens\n",
"\n",
" return len(answer_meaningful & context_meaningful) / len(answer_meaningful)\n",
"\n",
" def _grounding_check(self, answer: str, contexts: List[str]) -> Dict:\n",
" \"\"\"Sentence-level grounding assessment.\"\"\"\n",
" combined = \" \".join(contexts).lower()\n",
" sentences = [s.strip() for s in answer.split(\".\") if len(s.strip()) > 5]\n",
"\n",
" if not sentences:\n",
" return {\"grounded_ratio\": 0.0, \"total_sentences\": 0}\n",
"\n",
" grounded = 0\n",
" for sent in sentences:\n",
" words = set(sent.lower().split())\n",
" meaningful = words - {\"the\", \"a\", \"is\", \"are\", \"was\", \"it\", \"this\", \"that\", \"an\"}\n",
" if not meaningful:\n",
" grounded += 1\n",
" continue\n",
" ctx_words = set(combined.split())\n",
" ratio = len(meaningful & ctx_words) / len(meaningful)\n",
" if ratio > 0.4:\n",
" grounded += 1\n",
"\n",
" return {\n",
" \"grounded_ratio\": grounded / len(sentences),\n",
" \"total_sentences\": len(sentences),\n",
" \"grounded_sentences\": grounded,\n",
" }\n",
"\n",
" def _semantic_similarity(self, answer: str, contexts: List[str]) -> float:\n",
" \"\"\"Compute semantic similarity between answer and context.\"\"\"\n",
" if not self.embedding_model or not contexts:\n",
" return 0.0\n",
"\n",
" answer_emb = self.embedding_model.encode([answer], normalize_embeddings=True)\n",
" context_emb = self.embedding_model.encode(\n",
" [\" \".join(contexts)], normalize_embeddings=True\n",
" )\n",
" similarity = float(np.dot(answer_emb[0], context_emb[0]))\n",
" return similarity\n",
"\n",
" def _factual_indicators(self, answer: str, contexts: List[str]) -> Dict:\n",
" \"\"\"Check for factual consistency indicators.\"\"\"\n",
" answer_lower = answer.lower()\n",
" combined_context = \" \".join(contexts).lower()\n",
"\n",
" # Check for numbers in answer that aren't in context\n",
" import re\n",
" answer_numbers = set(re.findall(r'\\b\\d+\\.?\\d*\\b', answer_lower))\n",
" context_numbers = set(re.findall(r'\\b\\d+\\.?\\d*\\b', combined_context))\n",
" unsupported_numbers = answer_numbers - context_numbers\n",
"\n",
" # Check for proper nouns (capitalized words) not in context\n",
" answer_caps = set(re.findall(r'\\b[A-Z][a-z]+\\b', answer))\n",
" context_caps = set(re.findall(r'\\b[A-Z][a-z]+\\b', \" \".join(contexts)))\n",
" unsupported_entities = answer_caps - context_caps\n",
"\n",
" return {\n",
" \"unsupported_numbers\": list(unsupported_numbers),\n",
" \"unsupported_entities\": list(unsupported_entities)[:5],\n",
" \"has_unsupported_facts\": len(unsupported_numbers) > 0 or len(unsupported_entities) > 2,\n",
" }\n",
"\n",
" def _classify(self, signals: Dict) -> Dict:\n",
" \"\"\"Classify hallucination type based on aggregated signals.\"\"\"\n",
" overlap = signals.get(\"context_overlap\", 0)\n",
" grounding = signals.get(\"grounding\", {}).get(\"grounded_ratio\", 0)\n",
" semantic_sim = signals.get(\"semantic_similarity\", overlap) # fallback\n",
" has_unsupported = signals.get(\"factual_indicators\", {}).get(\"has_unsupported_facts\", False)\n",
"\n",
" # Weighted confidence\n",
" confidence = (0.3 * overlap + 0.3 * grounding + 0.2 * semantic_sim + 0.2 * (0.0 if has_unsupported else 1.0))\n",
"\n",
" # Classification logic\n",
" if confidence > 0.7:\n",
" return {\"type\": \"faithful\", \"confidence\": confidence}\n",
" elif has_unsupported and overlap < 0.4:\n",
" return {\"type\": \"factual\", \"confidence\": confidence}\n",
" elif grounding < 0.4:\n",
" return {\"type\": \"contextual\", \"confidence\": confidence}\n",
" else:\n",
" return {\"type\": \"reasoning\", \"confidence\": confidence}\n",
"\n",
"\n",
"# Initialize detector\n",
"hallucination_detector = HallucinationDetector(embedding_model=retrieval_system.embedding_model)\n",
"print(\" \u2713 Hallucination detector initialized\")\n",
"print(\" Signals: context_overlap, grounding_check, semantic_similarity, factual_indicators\")\n",
"print(\" Classifications: faithful, factual, contextual, reasoning\")\n",
"print()\n"
],
"outputs": [],
"execution_count": null
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Section 9: Logging Agent Demo\n",
"\n",
"Demonstrates structured logging with `[PLANNER]`, `[EXECUTOR]`, `[CRITIC]`, `[LOG]` prefixes. \n",
"All logs stored in `logger_agent.get_logs()` for later evaluation."
]
},
{
"cell_type": "code",
"metadata": {},
"source": [
"# \u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n",
"# \u2551 SECTION 9: LOGGING AGENT (STRUCTURED LOGGING) \u2551\n",
"# \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n",
"\n",
"# LoggingAgent already defined in Section 6.\n",
"# Here we demonstrate its structured output.\n",
"\n",
"print(\"=\" * 70)\n",
"print(\"SECTION 9: Logging Agent Demo\")\n",
"print(\"=\" * 70)\n",
"print()\n",
"\n",
"# Quick demo of the logging system\n",
"logger_agent.log(\"LOG\", \"System initialized\", {\"models\": len(available_models), \"kb_size\": len(knowledge_base)})\n",
"logger_agent.log(\"LOG\", \"Ready for evaluation\")\n",
"print()\n"
],
"outputs": [],
"execution_count": null
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Section 10: Dynamic Knowledge Update\n",
"\n",
"Demonstrates adding new documents to the knowledge base at runtime.\n",
"\n",
"The `add_new_document()` function:\n",
"- Chunks text into ~3-sentence segments\n",
"- Generates embeddings\n",
"- Updates FAISS index\n",
"- Rebuilds BM25 index\n",
"\n",
"> \u26a0\ufe0f **The model is NOT retrained. Only the retrieval knowledge base is updated.**"
]
},
{
"cell_type": "code",
"metadata": {},
"source": [
"# \u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n",
"# \u2551 SECTION 10: DYNAMIC KNOWLEDGE UPDATE \u2551\n",
"# \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n",
"\n",
"print(\"=\" * 70)\n",
"print(\"SECTION 10: Dynamic Knowledge Update\")\n",
"print(\"=\" * 70)\n",
"print()\n",
"print(\"NOTE: The model is NOT retrained. Only the retrieval knowledge base is updated.\")\n",
"print()\n",
"\n",
"\n",
"def add_new_document(text: str, retrieval_sys: RetrievalSystem = None):\n",
" \"\"\"\n",
" Add a new document to the knowledge base.\n",
" Chunks the text, generates embeddings, and updates both FAISS and BM25.\n",
"\n",
" NOTE: The model is not retrained. Only the retrieval knowledge base is updated.\n",
" \"\"\"\n",
" if retrieval_sys is None:\n",
" retrieval_sys = retrieval_system\n",
"\n",
" # Simple chunking: split by sentences, group into chunks of ~3 sentences\n",
" sentences = [s.strip() for s in text.replace(\"\\n\", \" \").split(\".\") if len(s.strip()) > 10]\n",
"\n",
" chunks = []\n",
" chunk_size = 3\n",
" for i in range(0, len(sentences), chunk_size):\n",
" chunk = \". \".join(sentences[i:i + chunk_size]) + \".\"\n",
" chunks.append(chunk)\n",
"\n",
" if not chunks:\n",
" chunks = [text]\n",
"\n",
" # Add each chunk to the retrieval system\n",
" for chunk in chunks:\n",
" retrieval_sys.add_document(chunk, metadata={\"source\": \"dynamic_update\"})\n",
"\n",
" print(f\" \u2713 Added {len(chunks)} chunk(s) to knowledge base\")\n",
" print(f\" \u2713 Knowledge base size: {len(retrieval_sys.documents)} documents\")\n",
" return chunks\n",
"\n",
"\n",
"# Demonstrate dynamic knowledge update\n",
"demo_text = \"\"\"\n",
"Large Language Models (LLMs) are neural networks with billions of parameters trained on massive\n",
"text corpora. Recent advances include mixture-of-experts architectures, which route different\n",
"inputs to specialized sub-networks. This allows scaling model capacity without proportionally\n",
"increasing compute costs. Models like Mixtral use 8 experts with a router that selects 2 experts\n",
"per token, achieving strong performance with lower inference cost than dense models of similar\n",
"total parameter count.\n",
"\"\"\"\n",
"\n",
"print(\" Adding new document about LLMs and Mixture of Experts...\")\n",
"new_chunks = add_new_document(demo_text)\n",
"print()\n",
"\n",
"# Test retrieval after update\n",
"test_results = retrieval_tool(\"What is mixture of experts in LLMs?\", strategy=\"hybrid\", k=3)\n",
"if test_results:\n",
" print(f\" \u2713 Verification: retrieved {len(test_results)} docs for MoE query\")\n",
" print(f\" Top result score: {test_results[0]['score']:.4f}\")\n",
"print()\n"
],
"outputs": [],
"execution_count": null
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Section 11: Full Evaluation Loop\n",
"\n",
"Runs the complete pipeline over all 55 queries \u00d7 3 retrieval strategies \u00d7 3 models.\n",
"\n",
"Collects per-run: retrieval score, context overlap, answer length, hallucination type, confidence, latency."
]
},
{
"cell_type": "code",
"metadata": {},
"source": [
"# \u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n",
"# \u2551 SECTION 11: EVALUATION LOOP \u2551\n",
"# \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n",
"\n",
"print(\"=\" * 70)\n",
"print(\"SECTION 11: Full Evaluation Loop\")\n",
"print(\"=\" * 70)\n",
"print()\n",
"\n",
"# Clear logs for evaluation\n",
"logger_agent.clear()\n",
"\n",
"\n",
"def run_full_pipeline(\n",
" question: str,\n",
" retrieval_strategy: str,\n",
" model_tuple: tuple,\n",
" retrieval_sys: RetrievalSystem,\n",
" detector: HallucinationDetector,\n",
" logger: LoggingAgent,\n",
") -> Dict:\n",
" \"\"\"\n",
" Run the complete RAG pipeline for a single query.\n",
" Returns all metrics for evaluation.\n",
" \"\"\"\n",
" start_time = time.time()\n",
" model_name, model = model_tuple\n",
"\n",
" # Step 1: Plan\n",
" plan = planner_agent.plan(question, retrieval_strategy)\n",
"\n",
" # Step 2: Retrieve\n",
" if retrieval_strategy == \"hybrid\":\n",
" retrieved = retrieval_sys.retrieve_hybrid(question, k=5)\n",
" elif retrieval_strategy == \"dense\":\n",
" retrieved = retrieval_sys.retrieve_dense(question, k=5)\n",
" elif retrieval_strategy == \"two_stage\":\n",
" retrieved = retrieval_sys.retrieve_two_stage(question, k=5)\n",
" else:\n",
" retrieved = retrieval_sys.retrieve_hybrid(question, k=5)\n",
"\n",
" contexts = [doc[\"document\"] for doc in retrieved]\n",
" retrieval_score = np.mean([doc[\"score\"] for doc in retrieved]) if retrieved else 0.0\n",
"\n",
" # Step 3: Generate\n",
" if hasattr(model, \"pipe\") and model.pipe and hasattr(model.pipe, \"tokenizer\"):\n",
" prompt = build_chat_prompt(question, contexts, model.pipe.tokenizer)\n",
" else:\n",
" prompt = build_rag_prompt(question, contexts)\n",
"\n",
" answer = model.generate(prompt, max_new_tokens=200)\n",
"\n",
" # Step 4: Detect hallucination\n",
" detection = detector.detect(answer, contexts, question)\n",
"\n",
" # Step 5: Compute metrics\n",
" total_latency = time.time() - start_time\n",
"\n",
" result = {\n",
" \"question\": question,\n",
" \"retrieval_strategy\": retrieval_strategy,\n",
" \"model_name\": model_name,\n",
" \"retrieved_docs\": contexts[:3], # Store top 3 for logging\n",
" \"retrieval_score\": retrieval_score,\n",
" \"generated_answer\": answer,\n",
" \"context_overlap\": detection[\"signals\"][\"context_overlap\"],\n",
" \"answer_length\": len(answer.split()),\n",
" \"hallucination_type\": detection[\"classification\"],\n",
" \"confidence_score\": detection[\"confidence\"],\n",
" \"is_hallucination\": detection[\"is_hallucination\"],\n",
" \"latency\": total_latency,\n",
" }\n",
"\n",
" # Log the result\n",
" logger.log(\"LOG\", f\"Pipeline complete: {retrieval_strategy}/{model_name}\", {\n",
" \"hallucination\": detection[\"classification\"],\n",
" \"confidence\": round(detection[\"confidence\"], 3),\n",
" \"latency_s\": round(total_latency, 3),\n",
" })\n",
"\n",
" return result\n",
"\n",
"\n",
"# Run evaluation\n",
"strategies = [\"hybrid\", \"dense\", \"two_stage\"]\n",
"all_results = []\n",
"\n",
"print(f\"Running evaluation: {len(eval_dataset)} queries \u00d7 {len(strategies)} strategies \u00d7 {len(available_models)} models\")\n",
"print(f\"Total runs: {len(eval_dataset) * len(strategies) * len(available_models)}\")\n",
"print()\n",
"\n",
"# Limit detailed logging during bulk evaluation\n",
"original_log = logger_agent.log\n",
"\n",
"\n",
"def quiet_log(agent, action, details=None):\n",
" \"\"\"Quiet logging during bulk eval (no print).\"\"\"\n",
" entry = {\n",
" \"timestamp\": time.time(),\n",
" \"agent\": agent,\n",
" \"action\": action,\n",
" \"details\": details or {},\n",
" }\n",
" logger_agent.logs.append(entry)\n",
"\n",
"\n",
"logger_agent.log = quiet_log # Suppress prints during bulk eval\n",
"\n",
"eval_count = 0\n",
"total_runs = len(eval_dataset) * len(strategies) * len(available_models)\n",
"\n",
"for strategy in strategies:\n",
" for model_tuple in available_models:\n",
" model_name = model_tuple[0]\n",
" for item in eval_dataset:\n",
" result = run_full_pipeline(\n",
" question=item[\"question\"],\n",
" retrieval_strategy=strategy,\n",
" model_tuple=model_tuple,\n",
" retrieval_sys=retrieval_system,\n",
" detector=hallucination_detector,\n",
" logger=logger_agent,\n",
" )\n",
" result[\"test_id\"] = item[\"id\"]\n",
" result[\"domain\"] = item[\"domain\"]\n",
" result[\"difficulty\"] = item[\"difficulty\"]\n",
" result[\"expected_hallucination_type\"] = item[\"expected_hallucination_type\"]\n",
" all_results.append(result)\n",
"\n",
" eval_count += 1\n",
" if eval_count % 50 == 0:\n",
" print(f\" Progress: {eval_count}/{total_runs} ({100*eval_count/total_runs:.0f}%)\")\n",
"\n",
"# Restore logging\n",
"logger_agent.log = original_log\n",
"\n",
"print(f\"\\n \u2713 Evaluation complete: {len(all_results)} total results\")\n",
"print()\n",
"\n",
"# Convert to DataFrame for analysis\n",
"results_df = pd.DataFrame(all_results)\n"
],
"outputs": [],
"execution_count": null
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Section 12: Evaluation Plots (Matplotlib)\n",
"\n",
"6 plots generated:\n",
"1. Hallucination type distribution\n",
"2. Confidence score histogram\n",
"3. Retrieval strategy vs hallucination rate\n",
"4. Retrieval strategy vs latency\n",
"5. Model comparison (hallucination rate)\n",
"6. Average confidence by domain"
]
},
{
"cell_type": "code",
"metadata": {},
"source": [
"# \u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n",
"# \u2551 SECTION 12: PLOTS (MATPLOTLIB ONLY) \u2551\n",
"# \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n",
"\n",
"print(\"=\" * 70)\n",
"print(\"SECTION 12: Generating Plots\")\n",
"print(\"=\" * 70)\n",
"print()\n",
"\n",
"fig, axes = plt.subplots(3, 2, figsize=(14, 16))\n",
"fig.suptitle(\"HALT-RAG: Evaluation Results\", fontsize=16, fontweight=\"bold\", y=0.98)\n",
"\n",
"# Plot 1: Hallucination Type Distribution\n",
"ax = axes[0, 0]\n",
"hall_counts = results_df[\"hallucination_type\"].value_counts()\n",
"ax.bar(hall_counts.index, hall_counts.values)\n",
"ax.set_title(\"1. Hallucination Type Distribution\")\n",
"ax.set_xlabel(\"Hallucination Type\")\n",
"ax.set_ylabel(\"Count\")\n",
"ax.tick_params(axis=\"x\", rotation=15)\n",
"for i, (label, count) in enumerate(zip(hall_counts.index, hall_counts.values)):\n",
" ax.text(i, count + 0.5, str(count), ha=\"center\", fontsize=9)\n",
"\n",
"# Plot 2: Confidence Score Histogram\n",
"ax = axes[0, 1]\n",
"ax.hist(results_df[\"confidence_score\"], bins=20, edgecolor=\"black\", alpha=0.7)\n",
"ax.set_title(\"2. Confidence Score Distribution\")\n",
"ax.set_xlabel(\"Confidence Score\")\n",
"ax.set_ylabel(\"Frequency\")\n",
"ax.axvline(results_df[\"confidence_score\"].mean(), color=\"red\", linestyle=\"--\",\n",
" label=f'Mean: {results_df[\"confidence_score\"].mean():.3f}')\n",
"ax.legend()\n",
"\n",
"# Plot 3: Retrieval Strategy vs Hallucination Rate\n",
"ax = axes[1, 0]\n",
"hall_rate_by_strategy = results_df.groupby(\"retrieval_strategy\")[\"is_hallucination\"].mean()\n",
"ax.bar(hall_rate_by_strategy.index, hall_rate_by_strategy.values)\n",
"ax.set_title(\"3. Retrieval Strategy vs Hallucination Rate\")\n",
"ax.set_xlabel(\"Strategy\")\n",
"ax.set_ylabel(\"Hallucination Rate\")\n",
"ax.set_ylim(0, 1)\n",
"for i, (label, rate) in enumerate(zip(hall_rate_by_strategy.index, hall_rate_by_strategy.values)):\n",
" ax.text(i, rate + 0.02, f\"{rate:.2%}\", ha=\"center\", fontsize=9)\n",
"\n",
"# Plot 4: Retrieval Strategy vs Latency\n",
"ax = axes[1, 1]\n",
"latency_by_strategy = results_df.groupby(\"retrieval_strategy\")[\"latency\"].agg([\"mean\", \"std\"])\n",
"ax.bar(latency_by_strategy.index, latency_by_strategy[\"mean\"],\n",
" yerr=latency_by_strategy[\"std\"], capsize=5)\n",
"ax.set_title(\"4. Retrieval Strategy vs Latency\")\n",
"ax.set_xlabel(\"Strategy\")\n",
"ax.set_ylabel(\"Latency (seconds)\")\n",
"for i, (label, row) in enumerate(latency_by_strategy.iterrows()):\n",
" ax.text(i, row[\"mean\"] + row[\"std\"] + 0.01, f'{row[\"mean\"]:.3f}s', ha=\"center\", fontsize=9)\n",
"\n",
"# Plot 5: Model Comparison (Hallucination Rate)\n",
"ax = axes[2, 0]\n",
"hall_rate_by_model = results_df.groupby(\"model_name\")[\"is_hallucination\"].mean()\n",
"ax.barh(hall_rate_by_model.index, hall_rate_by_model.values)\n",
"ax.set_title(\"5. Model Comparison: Hallucination Rate\")\n",
"ax.set_xlabel(\"Hallucination Rate\")\n",
"ax.set_xlim(0, 1)\n",
"for i, (label, rate) in enumerate(zip(hall_rate_by_model.index, hall_rate_by_model.values)):\n",
" ax.text(rate + 0.02, i, f\"{rate:.2%}\", va=\"center\", fontsize=9)\n",
"\n",
"# Plot 6 (Bonus): Domain-level Performance\n",
"ax = axes[2, 1]\n",
"domain_confidence = results_df.groupby(\"domain\")[\"confidence_score\"].mean().sort_values()\n",
"ax.barh(domain_confidence.index, domain_confidence.values)\n",
"ax.set_title(\"6. Average Confidence by Domain\")\n",
"ax.set_xlabel(\"Mean Confidence Score\")\n",
"for i, (label, conf) in enumerate(zip(domain_confidence.index, domain_confidence.values)):\n",
" ax.text(conf + 0.01, i, f\"{conf:.3f}\", va=\"center\", fontsize=9)\n",
"\n",
"plt.tight_layout(rect=[0, 0, 1, 0.96])\n",
"plt.savefig(\"halt_rag_results.png\", dpi=150, bbox_inches=\"tight\")\n",
"plt.show()\n",
"print(\" \u2713 Plots saved to 'halt_rag_results.png'\")\n",
"print()\n"
],
"outputs": [],
"execution_count": null
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Section 13: Final Summary & Analysis\n",
"\n",
"Prints:\n",
"- Best retrieval strategy (lowest hallucination rate)\n",
"- Best model\n",
"- Key observations\n",
"- Explicit limitations and caveats"
]
},
{
"cell_type": "code",
"metadata": {},
"source": [
"# \u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n",
"# \u2551 SECTION 13: OUTPUT + SUMMARY \u2551\n",
"# \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n",
"\n",
"print(\"=\" * 70)\n",
"print(\"SECTION 13: Final Summary & Analysis\")\n",
"print(\"=\" * 70)\n",
"print()\n",
"\n",
"# \u2500\u2500 Best Retrieval Strategy \u2500\u2500\n",
"print(\"\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\")\n",
"print(\"\u2502 RESULTS SUMMARY \u2502\")\n",
"print(\"\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\")\n",
"\n",
"# Strategy analysis\n",
"strategy_metrics = results_df.groupby(\"retrieval_strategy\").agg({\n",
" \"is_hallucination\": \"mean\",\n",
" \"confidence_score\": \"mean\",\n",
" \"latency\": \"mean\",\n",
" \"retrieval_score\": \"mean\",\n",
"}).round(4)\n",
"\n",
"print(\"\u2502\")\n",
"print(\"\u2502 Retrieval Strategy Comparison:\")\n",
"print(\"\u2502 \" + \"-\" * 67)\n",
"print(f\"\u2502 {'Strategy':<15} {'Hall. Rate':<12} {'Confidence':<12} {'Latency(s)':<12} {'Retr. Score':<12}\")\n",
"print(\"\u2502 \" + \"-\" * 67)\n",
"for strategy in strategy_metrics.index:\n",
" row = strategy_metrics.loc[strategy]\n",
" print(f\"\u2502 {strategy:<15} {row['is_hallucination']:<12.4f} {row['confidence_score']:<12.4f} \"\n",
" f\"{row['latency']:<12.4f} {row['retrieval_score']:<12.4f}\")\n",
"print(\"\u2502\")\n",
"\n",
"best_strategy = strategy_metrics[\"is_hallucination\"].idxmin()\n",
"print(f\"\u2502 \u2605 Best Retrieval Strategy: {best_strategy}\")\n",
"print(f\"\u2502 (lowest hallucination rate: {strategy_metrics.loc[best_strategy, 'is_hallucination']:.2%})\")\n",
"print(\"\u2502\")\n",
"\n",
"# Model analysis\n",
"model_metrics = results_df.groupby(\"model_name\").agg({\n",
" \"is_hallucination\": \"mean\",\n",
" \"confidence_score\": \"mean\",\n",
" \"latency\": \"mean\",\n",
"}).round(4)\n",
"\n",
"print(\"\u2502 Model Comparison:\")\n",
"print(\"\u2502 \" + \"-\" * 67)\n",
"print(f\"\u2502 {'Model':<25} {'Hall. Rate':<12} {'Confidence':<12} {'Latency(s)':<12}\")\n",
"print(\"\u2502 \" + \"-\" * 67)\n",
"for model in model_metrics.index:\n",
" row = model_metrics.loc[model]\n",
" print(f\"\u2502 {model:<25} {row['is_hallucination']:<12.4f} {row['confidence_score']:<12.4f} \"\n",
" f\"{row['latency']:<12.4f}\")\n",
"print(\"\u2502\")\n",
"\n",
"best_model = model_metrics[\"is_hallucination\"].idxmin()\n",
"print(f\"\u2502 \u2605 Best Model: {best_model}\")\n",
"print(f\"\u2502 (lowest hallucination rate: {model_metrics.loc[best_model, 'is_hallucination']:.2%})\")\n",
"print(\"\u2502\")\n",
"\n",
"# Key Observations\n",
"print(\"\u2502 KEY OBSERVATIONS:\")\n",
"print(\"\u2502 \" + \"-\" * 67)\n",
"\n",
"avg_confidence = results_df[\"confidence_score\"].mean()\n",
"print(f\"\u2502 \u2022 Average confidence score across all runs: {avg_confidence:.4f}\")\n",
"\n",
"hall_dist = results_df[\"hallucination_type\"].value_counts(normalize=True)\n",
"most_common_hall = hall_dist.index[0]\n",
"print(f\"\u2502 \u2022 Most common classification: '{most_common_hall}' ({hall_dist.iloc[0]:.1%} of answers)\")\n",
"\n",
"avg_latency = results_df[\"latency\"].mean()\n",
"print(f\"\u2502 \u2022 Average pipeline latency: {avg_latency:.3f} seconds\")\n",
"\n",
"# Difficulty analysis\n",
"diff_hall = results_df.groupby(\"difficulty\")[\"is_hallucination\"].mean()\n",
"print(f\"\u2502 \u2022 Hallucination rate by difficulty:\")\n",
"for diff in [\"easy\", \"medium\", \"hard\"]:\n",
" if diff in diff_hall.index:\n",
" print(f\"\u2502 - {diff}: {diff_hall[diff]:.2%}\")\n",
"\n",
"print(\"\u2502\")\n",
"print(\"\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\")\n",
"print(\"\u2502 LIMITATIONS & CAVEATS \u2502\")\n",
"print(\"\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\")\n",
"print(\"\u2502 \u2502\")\n",
"print(\"\u2502 1. SYNTHETIC DATASET: All test cases are artificially constructed. \u2502\")\n",
"print(\"\u2502 Results may not generalize to real-world queries and documents. \u2502\")\n",
"print(\"\u2502 \u2502\")\n",
"print(\"\u2502 2. PROBABILISTIC DETECTION: Hallucination classification is \u2502\")\n",
"print(\"\u2502 heuristic-based (token overlap + semantic similarity). It is \u2502\")\n",
"print(\"\u2502 NOT ground truth \u2014 false positives and negatives will occur. \u2502\")\n",
"print(\"\u2502 \u2502\")\n",
"print(\"\u2502 3. MODEL BIAS: Small models (TinyLlama, DistilGPT2) have limited \u2502\")\n",
"print(\"\u2502 instruction-following ability. Results would differ significantly\u2502\")\n",
"print(\"\u2502 with larger models (7B+, 70B+). \u2502\")\n",
"print(\"\u2502 \u2502\")\n",
"print(\"\u2502 4. CONTROLLED RETRIEVAL: The knowledge base is derived from the \u2502\")\n",
"print(\"\u2502 test set itself. In production, retrieval noise would increase \u2502\")\n",
"print(\"\u2502 hallucination rates significantly. \u2502\")\n",
"print(\"\u2502 \u2502\")\n",
"print(\"\u2502 5. NO TRUE GROUND-TRUTH EVALUATION: Without human-annotated \u2502\")\n",
"print(\"\u2502 hallucination labels, we cannot compute true precision/recall \u2502\")\n",
"print(\"\u2502 for the detection system. \u2502\")\n",
"print(\"\u2502 \u2502\")\n",
"print(\"\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\")\n",
"print()\n",
"\n",
"# \u2500\u2500 Logging Summary \u2500\u2500\n",
"print(\"Logging Summary:\")\n",
"log_summary = logger_agent.get_summary()\n",
"print(f\" Total log entries: {log_summary['total_entries']}\")\n",
"print(f\" By agent: {dict(log_summary['by_agent'])}\")\n",
"print()\n",
"\n",
"# \u2500\u2500 Sample Logs \u2500\u2500\n",
"print(\"Sample Structured Logs (last 5):\")\n",
"for log_entry in logger_agent.get_logs()[-5:]:\n",
" ts = time.strftime(\"%H:%M:%S\", time.localtime(log_entry[\"timestamp\"]))\n",
" print(f\" [{ts}] [{log_entry['agent']:8s}] {log_entry['action']}\")\n",
"\n",
"print()\n",
"print(\"=\" * 70)\n",
"print(\"HALT-RAG Demo Complete\")\n",
"print(\"=\" * 70)\n",
"print()\n",
"print(\"Files generated:\")\n",
"print(\" \u2022 halt_rag_results.png \u2014 Evaluation plots\")\n",
"print()\n",
"print(\"To access full logs programmatically:\")\n",
"print(\" logs = logger_agent.get_logs()\")\n",
"print(\" results = results_df # pandas DataFrame with all evaluation results\")\n",
"print()\n",
"print(\"This is a graduate-level AI systems demo.\")\n",
"print(\"Clarity, correctness, and reproducibility > complexity.\")\n"
],
"outputs": [],
"execution_count": null
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"---\n",
"\n",
"## \u2705 Demo Complete\n",
"\n",
"**Access results programmatically:**\n",
"```python\n",
"logs = logger_agent.get_logs() # All structured logs\n",
"results = results_df # pandas DataFrame with all evaluation results\n",
"```\n",
"\n",
"**This is a graduate-level AI systems demo.** \n",
"Clarity, correctness, and reproducibility > complexity."
]
}
]
} |