Add Phase 2 case law: 20 Federal Court of Appeal and Federal Court decisions

canlex/caselaw.py

@@ -1,9 +1,11 @@
-"""Ingest leading Supreme Court of Canada decisions as section-style chunks.
+"""Ingest leading Canadian court decisions as section-style chunks.
 
-Source: the SCC's official decisions database (decisions.scc-csc.ca, the Lexum
-"Norma" platform). A decision's text sits inside an iframe, so each item is
-fetched by appending ?iframe=true to its URL. This ingests a *curated* set of
-leading cases -- it is deliberately not a comprehensive scrape.
+Sources: the official Lexum decisions databases of the Supreme Court of Canada
+(decisions.scc-csc.ca), the Federal Court of Appeal (decisions.fca-caf.gc.ca)
+and the Federal Court (decisions.fct-cf.gc.ca). A decision's text sits inside an
+iframe, so each item is fetched by appending ?iframe=true to its URL. This
+ingests a *curated* set of leading cases -- it is deliberately not a
+comprehensive scrape.
 
     py -m canlex.caselaw
 """
@@ -16,13 +18,23 @@ from bs4 import BeautifulSoup
 
 from .config import PROCESSED_DIR, RAW_DIR
 
-ITEM_URL = "https://decisions.scc-csc.ca/scc-csc/scc-csc/en/item/{id}/index.do"
-_RAW = RAW_DIR / "scc"
+# Each court's official Lexum database: (display name, item-URL template). All
+# three sites behave identically -- same iframe trick, metadata block and
+# bracketed paragraph numbers -- so one parser serves them all.
+COURTS = {
+    "scc": ("Supreme Court of Canada",
+            "https://decisions.scc-csc.ca/scc-csc/scc-csc/en/item/{id}/index.do"),
+    "fca": ("Federal Court of Appeal",
+            "https://decisions.fca-caf.gc.ca/fca-caf/decisions/en/item/{id}/index.do"),
+    "fc": ("Federal Court",
+           "https://decisions.fct-cf.gc.ca/fc-cf/decisions/en/item/{id}/index.do"),
+}
+_RAW = RAW_DIR / "caselaw"
 OUT = PROCESSED_DIR / "caselaw.json"
 
-# A normal browser User-Agent: the SCC site denylists a few crawler UAs, while
-# its robots.txt otherwise permits the decision pages. Politeness comes from the
-# throttle below and from caching every fetched page on disk.
+# A normal browser User-Agent: the courts' Lexum sites denylist a few crawler
+# UAs, while robots.txt otherwise permits the decision pages. Politeness comes
+# from the throttle below and from caching every fetched page on disk.
 _UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
        "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
 _THROTTLE = 2.0       # seconds between live fetches
@@ -32,79 +44,146 @@ _CHUNK_CHARS = 1800   # target characters per chunk
 # which is not part of the judgment's reasons.
 _APPARATUS = re.compile(r"^\s*(APPENDIX\b|Solicitors?\s+for\b)", re.I)
 
-# Curated leading SCC cases on border / immigration / customs / Charter law.
-# 'id' is the verified decisions.scc-csc.ca item ID; 'short' and 'topic' are
-# curated. The case name, citation and date are parsed from the page itself.
-SCC_CASES = [
-    {"id": 18078, "short": "Vavilov",
+# Curated leading cases on Canadian border / immigration / customs / Charter
+# law. 'court' selects a database in COURTS; 'id' is the verified Lexum item ID;
+# 'short' and 'topic' are curated. The case name, citation and date are parsed
+# from the page. FCA decisions later reversed by the SCC are deliberately
+# omitted -- CanLex serves good law, and the SCC versions are already listed.
+CASES = [
+    # --- Supreme Court of Canada ---
+    {"court": "scc", "id": 18078, "short": "Vavilov",
      "topic": "Standard of review on judicial review; the reasonableness "
               "standard for administrative decisions"},
-    {"id": 20081, "short": "Mason",
+    {"court": "scc", "id": 20081, "short": "Mason",
      "topic": "Inadmissibility under IRPA s. 34(1)(e) for acts of violence "
               "endangering safety in Canada; reasonableness review"},
-    {"id": 16803, "short": "Tran",
+    {"court": "scc", "id": 16803, "short": "Tran",
      "topic": "Serious criminality and inadmissibility under IRPA s. 36; the "
               "meaning of a term of imprisonment and an offence punishable by"},
-    {"id": 15647, "short": "B010",
+    {"court": "scc", "id": 15647, "short": "B010",
      "topic": "Inadmissibility for people smuggling under IRPA s. 37(1)(b); "
               "organized criminality"},
-    {"id": 15648, "short": "Appulonappa",
+    {"court": "scc", "id": 15648, "short": "Appulonappa",
      "topic": "The human smuggling offence in IRPA s. 117; constitutional "
               "overbreadth and humanitarian aid to asylum seekers"},
-    {"id": 14419, "short": "Febles",
+    {"court": "scc", "id": 14419, "short": "Febles",
      "topic": "Exclusion from refugee protection for a serious non-political "
               "crime under Article 1F(b) of the Refugee Convention"},
-    {"id": 13184, "short": "Ezokola",
+    {"court": "scc", "id": 13184, "short": "Ezokola",
      "topic": "Complicity and exclusion from refugee protection for "
               "international crimes under Article 1F(a)"},
-    {"id": 15665, "short": "Kanthasamy",
+    {"court": "scc", "id": 15665, "short": "Kanthasamy",
      "topic": "Humanitarian and compassionate relief under IRPA s. 25; the "
               "best interests of a child"},
-    {"id": 13137, "short": "Agraira",
+    {"court": "scc", "id": 13137, "short": "Agraira",
      "topic": "Ministerial relief from inadmissibility on security grounds; "
               "the national interest under IRPA"},
-    {"id": 6901, "short": "Khosa",
+    {"court": "scc", "id": 6901, "short": "Khosa",
      "topic": "Standard of review of immigration decisions; judicial review "
               "of a removal order"},
-    {"id": 2345, "short": "Charkaoui",
+    {"court": "scc", "id": 2345, "short": "Charkaoui",
      "topic": "Security certificates and immigration detention; the Charter "
               "and procedural fairness"},
-    {"id": 1937, "short": "Suresh",
+    {"court": "scc", "id": 1937, "short": "Suresh",
      "topic": "Deportation to a risk of torture; Charter s. 7 and removal on "
               "security grounds"},
-    {"id": 17759, "short": "Chhina",
+    {"court": "scc", "id": 17759, "short": "Chhina",
      "topic": "Habeas corpus as a remedy for immigration detention; review of "
               "lengthy detention"},
-    {"id": 1717, "short": "Baker",
+    {"court": "scc", "id": 1717, "short": "Baker",
      "topic": "Procedural fairness in administrative decisions; humanitarian "
               "and compassionate review; the duty to give reasons"},
-    {"id": 39, "short": "Singh",
+    {"court": "scc", "id": 39, "short": "Singh",
      "topic": "Charter s. 7 rights of refugee claimants; the right to an oral "
               "hearing"},
-    {"id": 377, "short": "Simmons",
+    {"court": "scc", "id": 377, "short": "Simmons",
      "topic": "Customs searches at the border; Charter s. 8 and the reasonable "
               "expectation of privacy on entry to Canada"},
-    {"id": 1694, "short": "Monney",
+    {"court": "scc", "id": 1694, "short": "Monney",
      "topic": "Border detention for a customs search; reasonable suspicion "
               "and the Customs Act"},
-    {"id": 986, "short": "Dehghani",
+    {"court": "scc", "id": 986, "short": "Dehghani",
      "topic": "Charter rights at a port of entry; secondary examination and "
               "the right to counsel"},
-    {"id": 1627, "short": "Pushpanathan",
+    {"court": "scc", "id": 1627, "short": "Pushpanathan",
      "topic": "Exclusion from refugee protection under Article 1F(c) for acts "
               "contrary to the purposes of the United Nations"},
-    {"id": 1023, "short": "Ward",
+    {"court": "scc", "id": 1023, "short": "Ward",
      "topic": "The refugee definition; a particular social group; the "
               "availability of state protection"},
+
+    # --- Federal Court of Appeal ---
+    {"court": "fca", "id": 143136, "short": "Huruglica",
+     "topic": "The Refugee Appeal Division reviews the Refugee Protection "
+              "Division for correctness and makes its own independent assessment"},
+    {"court": "fca", "id": 143152, "short": "Singh (new evidence)",
+     "topic": "Admitting new evidence at the Refugee Appeal Division under "
+              "IRPA s. 110(4)"},
+    {"court": "fca", "id": 37663, "short": "Hernandez Febles",
+     "topic": "Exclusion from refugee protection for a serious non-political "
+              "crime, Refugee Convention Article 1F(b)"},
+    {"court": "fca", "id": 36253, "short": "Jayasekara",
+     "topic": "The test for a serious crime under Article 1F(b) exclusion "
+              "from refugee protection"},
+    {"court": "fca", "id": 35786, "short": "Raza",
+     "topic": "Pre-removal risk assessment; the test for new evidence under "
+              "IRPA s. 113(a)"},
+    {"court": "fca", "id": 99694, "short": "Najafi",
+     "topic": "Security inadmissibility under IRPA s. 34(1)(b), subversion "
+              "against a government"},
+    {"court": "fca", "id": 108889, "short": "Kanagendren",
+     "topic": "Security inadmissibility for membership under IRPA s. 34(1)(f); "
+              "membership requires no complicity analysis"},
+    {"court": "fca", "id": 35313, "short": "Sittampalam",
+     "topic": "Organized criminality under IRPA s. 37(1)(a); the broad meaning "
+              "of a criminal organization"},
+    {"court": "fca", "id": 31607, "short": "Thanabalasingham",
+     "topic": "Immigration detention review; the Minister's onus and the "
+              "weight owed to prior detention decisions"},
+    {"court": "fca", "id": 305100, "short": "Lunyamila",
+     "topic": "Immigration detention where a detainee will not cooperate with "
+              "their own removal"},
+    {"court": "fca", "id": 483607, "short": "Brown",
+     "topic": "The constitutionality of immigration detention; the Charter "
+              "and lengthy detention"},
+    {"court": "fca", "id": 36347, "short": "Baron",
+     "topic": "Deferral of removal by an enforcement officer; the narrow "
+              "scope of the discretion to defer"},
+    {"court": "fca", "id": 520921, "short": "Galindo Camayo",
+     "topic": "Reasonableness review of an immigration decision and the role "
+              "of certified questions on appeal"},
+    {"court": "fca", "id": 501244, "short": "Best Buy",
+     "topic": "Customs tariff classification; methodology and appeals from "
+              "the Canadian International Trade Tribunal"},
+    {"court": "fca", "id": 466027, "short": "Honey Fashions",
+     "topic": "Customs; judicial review of the Canada Border Services "
+              "Agency's exercise of discretion"},
+    {"court": "fca", "id": 419470, "short": "Hociung",
+     "topic": "Customs Act seizure and forfeiture; the meaning of imported "
+              "goods and their valuation"},
+
+    # --- Federal Court ---
+    {"court": "fc", "id": 64594, "short": "Goburdhun",
+     "topic": "Inadmissibility for misrepresentation under IRPA s. 40; a "
+              "frequently-cited summary of the governing principles"},
+    {"court": "fc", "id": 492842, "short": "Garcia",
+     "topic": "Criminal inadmissibility; the effect of withdrawn charges and "
+              "a claim of self-defence"},
+    {"court": "fc", "id": 483303, "short": "Kaur",
+     "topic": "Procedural fairness; the right to a meaningful opportunity to "
+              "answer a visa officer's concerns"},
+    {"court": "fc", "id": 56900, "short": "Nguyen",
+     "topic": "Customs Act; judicial review of the amount set for a penalty "
+              "or ascertained forfeiture"},
 ]
 
 
-def _fetch(item_id):
+def _fetch(court, item_id):
     """Return a decision's iframe HTML, caching the raw page under data/raw."""
-    cache = _RAW / f"{item_id}.html"
+    cache = _RAW / f"{court}-{item_id}.html"
     if cache.exists():
         return cache.read_text(encoding="utf-8")
-    url = ITEM_URL.format(id=item_id) + "?iframe=true"
+    url = COURTS[court][1].format(id=item_id) + "?iframe=true"
     req = urllib.request.Request(url, headers={"User-Agent": _UA})
     time.sleep(_THROTTLE)
     with urllib.request.urlopen(req, timeout=60) as resp:
@@ -146,10 +225,10 @@ def _body(soup):
 def _paragraphs(soup):
     """Return (is_numbered, [(label, text), ...]) for the judgment body.
 
-    Modern SCC judgments open each paragraph with a bracketed number "[N]".
-    They are detected by content -- a run of sequentially numbered <p> blocks --
-    so the parser does not depend on Word style names, which vary by era. Every
-    <p> between one numbered opener and the next belongs to that paragraph.
+    Modern judgments open each paragraph with a bracketed number "[N]". They are
+    detected by content -- a run of sequentially numbered <p> blocks -- so the
+    parser does not depend on Word style names, which vary by court and era.
+    Every <p> between one numbered opener and the next belongs to that paragraph.
     Older, unnumbered decisions fall back to taking every <p> in document order.
     """
     blocks = [p for p in _body(soup).find_all("p")
@@ -238,13 +317,14 @@ def _chunk(paras):
 
 def _decision_chunks(case, soup):
     """Build CanLex chunk dicts for one decision."""
+    court_name, item_tmpl = COURTS[case["court"]]
     name, fields = _metadata(soup)
     name = name or case["short"]
     cite = fields.get("neutral citation") or fields.get("report") or ""
     report = fields.get("report", "")
     date = fields.get("date", "")
     citation = f"{name}, {cite}" if cite else name
-    item_url = ITEM_URL.format(id=case["id"])
+    item_url = item_tmpl.format(id=case["id"])
     modern, paras = _paragraphs(soup)
     chunks = []
     for i, group in enumerate(_chunk(paras), start=1):
@@ -255,16 +335,16 @@ def _decision_chunks(case, soup):
         else:
             locator = f"excerpt {i}"
         chunks.append({
-            "id": f"scc-{case['id']}-{i}",
+            "id": f"{case['court']}-{case['id']}-{i}",
             "doc_type": "caselaw",
-            "act_code": cite or f"SCC item {case['id']}",
+            "act_code": cite or f"{case['court'].upper()} item {case['id']}",
             "act_short": case["short"],
             "act_name": name,
             "section": "",
             "citation": citation,
             "marginal_note": locator,
             "heading": case["topic"],
-            "part": "Supreme Court of Canada",
+            "part": court_name,
             "division": "",
             "text": "\n\n".join(t for _, t in group),
             "current_to": date,
@@ -276,28 +356,29 @@ def _decision_chunks(case, soup):
 
 
 def build():
-    """Fetch, parse and chunk every curated SCC decision into caselaw.json."""
+    """Fetch, parse and chunk every curated decision into caselaw.json."""
     all_chunks = []
-    for case in SCC_CASES:
+    for case in CASES:
         try:
-            soup = BeautifulSoup(_fetch(case["id"]), "html.parser")
+            soup = BeautifulSoup(_fetch(case["court"], case["id"]),
+                                 "html.parser")
         except Exception as exc:
             print(f"  !! {case['short']}: fetch failed -- "
                   f"{type(exc).__name__}: {exc}")
             continue
         chunks, citation, n_paras = _decision_chunks(case, soup)
         if not chunks:
-            print(f"  !! {case['short']} (item {case['id']}): "
-                  f"0 chunks -- check parsing")
+            print(f"  !! {case['short']} ({case['court']} item "
+                  f"{case['id']}): 0 chunks -- check parsing")
             continue
         all_chunks.extend(chunks)
-        print(f"  {case['short']:13s} {n_paras:4d} paras -> "
+        print(f"  {case['court']:4s} {case['short']:20s} {n_paras:4d} paras -> "
               f"{len(chunks):3d} chunks   {citation}")
     PROCESSED_DIR.mkdir(parents=True, exist_ok=True)
     OUT.write_text(json.dumps(all_chunks, ensure_ascii=False, indent=1),
                    encoding="utf-8")
     print(f"\n{len(all_chunks)} case-law chunks from "
-          f"{len(SCC_CASES)} SCC decisions -> {OUT}")
+          f"{len(CASES)} decisions -> {OUT}")
 
 
 if __name__ == "__main__":

canlex/server.py

@@ -125,8 +125,8 @@ class SearchInput(BaseModel):
         default=None,
         description="Optional filter by source type: 'legislation' (Acts and "
         "regulations), 'memorandum' (CBSA D-Memoranda), 'agreement' (collective "
-        "agreements), 'directive' (NJC directives), or 'caselaw' (Supreme Court "
-        "of Canada decisions). Omit to search all.",
+        "agreements), 'directive' (NJC directives), or 'caselaw' (Supreme Court, "
+        "Federal Court of Appeal and Federal Court decisions). Omit to search all.",
     )
 
 
@@ -144,7 +144,7 @@ class GetSectionInput(BaseModel):
           annotations={"title": "Search Canadian Legislation", **_READONLY})
 def canlex_search_legislation(params: SearchInput) -> str:
     """Search Canadian federal law, CBSA D-Memoranda, agreements, NJC directives,
-    and leading Supreme Court of Canada cases.
+    and leading court decisions.
 
     The CanLex corpus has five kinds of source: 31 federal Acts and regulations
     (immigration, customs, criminal, drugs, food/health, labour, privacy and more);
@@ -152,8 +152,8 @@ def canlex_search_legislation(params: SearchInput) -> str:
     how it applies customs and border law); Treasury Board collective agreements
     (currently the FB / Border Services group); National Joint Council directives
     (travel, relocation, isolated posts and more); and leading Supreme Court of
-    Canada decisions on immigration, customs and Charter-at-the-border law. Use this
-    for ANY question about that material. It ranks results by relevance and returns
+    Canada, Federal Court of Appeal and Federal Court decisions on immigration,
+    customs and Charter law. Use this for ANY question about that material. It ranks results by relevance and returns
     their full text so the answer can cite the actual wording; an explicit section
     reference (e.g. "section 34") is always surfaced. Each result is marked with its
     source type.
@@ -297,7 +297,7 @@ def canlex_list_acts() -> str:
             lines.append(f"- **{a['short']}**: {a['count']} sections, "
                          f"effective {a['current_to'] or 'n/a'}")
     if cases:
-        lines += ["", "## Case law (Supreme Court of Canada)"]
+        lines += ["", "## Case law"]
         for cite, a in sorted(cases.items(), key=lambda kv: kv[1]["decided"]):
             lines.append(f"- **{a['name']}**, {cite}: {a['count']} excerpts, "
                          f"decided {a['decided'] or 'n/a'}")