eval/questions.json

[
  {
    "id": "q01",
    "category": "list",
    "question": "How many documents are uploaded and what types are they?",
    "expected_tools": ["list_documents"],
    "expected_substrings": ["invoice", "contract"]
  },
  {
    "id": "q02",
    "category": "list",
    "question": "List the uploaded files",
    "expected_tools": ["list_documents"],
    "expected_substrings": [".pdf"]
  },
  {
    "id": "q03",
    "category": "extract",
    "question": "What is the gross total on as-2026-001.pdf?",
    "expected_tools": ["list_documents", "get_extraction"],
    "expected_substrings": ["24000", "24,000", "gross", "$"]
  },
  {
    "id": "q04",
    "category": "extract",
    "question": "Who issued as-2026-002.pdf?",
    "expected_tools": ["get_extraction"],
    "expected_substrings": ["AcmeSoft", "issuer"]
  },
  {
    "id": "q05",
    "category": "extract",
    "question": "What is the payment due date on bi-inv-2026-0418.pdf?",
    "expected_tools": ["get_extraction"],
    "expected_substrings": ["2026", "due"]
  },
  {
    "id": "q06",
    "category": "search",
    "question": "Which document has information on the delivery due date?",
    "expected_tools": ["search_documents"],
    "expected_substrings": ["purchase_order", "delivery"]
  },
  {
    "id": "q07",
    "category": "search",
    "question": "What is the penalty amount in the NDA? Find it.",
    "expected_tools": ["search_documents"],
    "expected_substrings": ["penalty", "50"]
  },
  {
    "id": "q08",
    "category": "search",
    "question": "What does the change of control clause contain?",
    "expected_tools": ["search_documents"],
    "expected_substrings": ["25", "ownership", "change"]
  },
  {
    "id": "q09",
    "category": "compare",
    "question": "Compare the prices of as-2026-001.pdf and as-2026-003.pdf.",
    "expected_tools": ["get_extraction", "compare_documents"],
    "expected_substrings": ["differ", "diff", "net"]
  },
  {
    "id": "q10",
    "category": "compare",
    "question": "How much more expensive is as-2026-003.pdf compared to as-2026-001.pdf?",
    "expected_tools": ["get_extraction", "compare_documents"],
    "expected_substrings": ["differ", "diff"]
  },
  {
    "id": "q11",
    "category": "compare",
    "question": "Is there a discrepancy between bi-po-2026-0412.pdf and bi-dn-2026-0415.pdf?",
    "expected_tools": ["compare_documents"],
    "expected_substrings": ["HI-100", "differ", "38", "40"]
  },
  {
    "id": "q12",
    "category": "validate",
    "question": "Validate the math on as-2026-001.pdf.",
    "expected_tools": ["validate_document"],
    "expected_substrings": ["ok", "error", "valid", "math"]
  },
  {
    "id": "q13",
    "category": "validate",
    "question": "Is there a math error in adv-inv-2026-0001.pdf?",
    "expected_tools": ["validate_document"],
    "expected_substrings": ["adv-inv-2026-0001", "error", "ok"]
  },
  {
    "id": "q14",
    "category": "validate",
    "question": "Is the tax ID valid on as-2026-002.pdf?",
    "expected_tools": ["validate_document"],
    "expected_substrings": ["tax", "ok", "12-3456789"]
  }
]