Shift Space to overview-first ranked relationships UX

public_copy.json

@@ -4,7 +4,7 @@
   "subtitle": "Neutral Records explorer for a public-record slice of congressional money-and-power linkages.",
   "dataset_repo_id": "cjc0013/cmp-data",
   "space_repo_id": "cjc0013/cmp",
-  "welcome_markdown": "# Congress Public Records Slice\n\nStart with the **Network Graph** tab.\n\n- Search one House member name first.\n- Green dots are House members, rust dots are funding recipients, and gold dots are sectors.\n- Lines show public-record support in this released slice; thicker lines mean more supporting rows.\n- Use **Explore** and **Event Detail** to inspect the underlying rows, source URLs, and SHA-backed artifacts.\n\nThis is an exploration tool, not an accusation tool.",
+  "welcome_markdown": "# Congress Public Records Slice\n\nStart with **Overview** for the clearest read.\n\n- Pick one House member first.\n- Use **Overview** to see the strongest sectors or funding recipients for that member.\n- Use **Explain Link** to see why one relationship appears in this released slice.\n- Use **Explore Graph** only if you want a secondary visual map.\n\nThis is an exploration tool, not an accusation tool.",
   "landing_markdown": "# Congress Public Records Slice\n\nA neutral, review-oriented slice of House public-record linkages across financial disclosures, sector overlap, and community project funding recipient relationships.\n\n- This release is a slice of public-record data, not a complete accounting of all potentially relevant data.\n- Future releases may update or expand this slice as source recovery, parsing, and evidence linkage improve.\n- This release does not assign guilt, wrongdoing, intent, or causality to any person or organization.\n- The release shows public-record overlaps, timing, and linkage strength, not proof of illegality or corruption.\n- Some rows remain review-tier or include unresolved official source references and should be read with those labels in mind.\n- The public package includes verification summaries and SHA-backed artifact indexes, but it does not include the full internal raw corpus, so external verification is bounded by what is published here.",
   "downloads_markdown": "## Downloads\n\n- Dataset repo id: `cjc0013/cmp-data`\n- Space repo id: `cjc0013/cmp`\n\nUse the dataset bundle files for direct review, CSV download, and SHA-backed source checks.",
   "dataset_bundle_prefix": "dataset_bundle"

public_space_app.py

@@ -3,9 +3,11 @@ from __future__ import annotations
 import html
 import json
 import os
+import re
 import urllib.request
 from pathlib import Path
 from typing import Any, Dict, Tuple
+from urllib.parse import urlparse
 
 import pandas as pd
 
@@ -166,8 +168,9 @@ def _graph_intro_markdown(config: Dict[str, Any]) -> str:
     )
     return "\n".join(
         [
-            "### What you are looking at",
+            "### Optional graph view",
             "",
+            "- Use this only after the overview if you want a visual map.",
             "- Green dots are House members, rust dots are funding recipients, and gold dots are sectors.",
             "- Thicker lines mean more supporting relationship rows in this released slice.",
             opening_line,
@@ -228,12 +231,105 @@ def _graph_view_summary_markdown(
     return "\n".join(lines)
 
 
-def _graph_table(edges: pd.DataFrame) -> pd.DataFrame:
+def _plain_reason_code(value: str) -> str:
+    normalized = str(value or "").strip()
+    mapping = {
+        "recipient_exact_match": "Exact recipient match",
+        "issuer_match": "Issuer or company match",
+        "legislative_relevance_match": "Legislative topic match",
+        "major_vote_overlap": "Vote activity overlaps the same topic window",
+        "lobbying_issue_overlap": "Lobbying activity overlaps the same topic window",
+        "legislative_density_support": "Many related bill records in the same area",
+        "vote_density_support": "Many related vote records in the same area",
+        "lobbying_density_support": "Many related lobbying filings in the same area",
+        "insufficient_official_support": "Not enough official support for a stronger label",
+    }
+    return mapping.get(normalized, normalized.replace("_", " ").title() or "Signal")
+
+
+def _edge_evidence_chips(row: Dict[str, Any]) -> list[str]:
+    urls = _split_pipe_values(row.get("source_urls", ""), limit=12)
+    reason_codes = set(_split_pipe_values(row.get("reason_codes", ""), limit=20))
+    chips: list[str] = []
+    if any("/ptr-pdfs/" in url for url in urls):
+        chips.append("trade disclosure")
+    if any("/financial-pdfs/" in url for url in urls):
+        chips.append("annual disclosure")
+    if any("govinfo.gov/bulkdata/BILLSTATUS" in url for url in urls):
+        chips.append("bill record")
+    if any("usaspending.gov/award/" in url for url in urls):
+        chips.append("funding award")
+    if any("committee_info" in url for url in urls):
+        chips.append("committee roster")
+    if "major_vote_overlap" in reason_codes or "vote_density_support" in reason_codes:
+        chips.append("vote activity")
+    if "lobbying_issue_overlap" in reason_codes or "lobbying_density_support" in reason_codes:
+        chips.append("lobbying activity")
+    if int(row.get("profile_link_count", 0) or 0) > 0:
+        chips.append("member profile")
+    deduped: list[str] = []
+    for chip in chips:
+        if chip not in deduped:
+            deduped.append(chip)
+    return deduped[:6]
+
+
+def _window_overlap_text(row: Dict[str, Any]) -> str:
+    reason_codes = set(_split_pipe_values(row.get("reason_codes", ""), limit=20))
+    overlap_signals = [code for code in reason_codes if "overlap" in code]
+    if overlap_signals:
+        count = len(overlap_signals)
+        return f"yes ({count} overlap signal{'s' if count != 1 else ''})"
+    if int(row.get("profile_link_count", 0) or 0) > 0:
+        return "profile support only"
+    if int(row.get("unresolved_source_ref_count", 0) or 0) > 0:
+        return "some timing still unresolved"
+    return "not explicit in this row"
+
+
+def _relationship_score(row: Dict[str, Any]) -> int:
+    status = str(row.get("relationship_status", "") or "")
+    family = str(row.get("relationship_family", "") or "")
+    stronger_support = int(
+        row.get("linked_count", 0) or 0
+        if family == "recipient"
+        else row.get("strong_event_count", 0) or 0
+    )
+    status_base = {
+        "linked": 78,
+        "release_ok": 74,
+        "acceptable_with_label": 56,
+        "needs_review": 44,
+        "unresolved": 20,
+    }.get(status, 30)
+    score = status_base
+    score += min(int(row.get("link_count", 0) or 0) * 3, 15)
+    score += min(stronger_support * 4, 18)
+    score += min(len(_edge_evidence_chips(row)) * 2, 10)
+    score -= min(int(row.get("unresolved_source_ref_count", 0) or 0), 12)
+    return max(0, min(100, score))
+
+
+def _rank_relationships(edges: pd.DataFrame) -> pd.DataFrame:
+    columns = [
+        "rank",
+        "relationship_id",
+        "member",
+        "counterparty / sector",
+        "overall score",
+        "strength",
+        "evidence",
+        "time-window overlap",
+        "supporting rows",
+        "stronger support",
+        "needs caution",
+        "unresolved refs",
+        "source_examples",
+    ]
     if edges.empty:
-        return pd.DataFrame(columns=["member", "target", "relationship_view", "strength", "supporting_rows"])
+        return pd.DataFrame(columns=columns)
     rows: list[dict[str, Any]] = []
     for row in edges.to_dict("records"):
-        status = str(row.get("relationship_status", "") or "")
         family = str(row.get("relationship_family", "") or "")
         stronger_support = int(
             row.get("linked_count", 0) or 0
@@ -245,21 +341,189 @@ def _graph_table(edges: pd.DataFrame) -> pd.DataFrame:
             if family == "recipient"
             else row.get("weak_event_count", 0) or 0
         )
-        source_examples = ", ".join(_split_pipe_values(row.get("source_urls", ""), limit=2))
+        chips = _edge_evidence_chips(row)
         rows.append(
             {
+                "relationship_id": str(row.get("edge_id") or ""),
                 "member": str(row.get("member_name") or row.get("member_slug") or ""),
-                "target": str(row.get("target_label") or ""),
-                "relationship_view": _plain_family_label(family),
-                "strength": _plain_status_label(status),
-                "supporting_rows": int(row.get("link_count", 0) or 0),
-                "stronger_support": stronger_support,
-                "caution_support": caution_support,
-                "unresolved_refs": int(row.get("unresolved_source_ref_count", 0) or 0),
-                "source_examples": source_examples,
+                "counterparty / sector": str(row.get("target_label") or ""),
+                "overall score": _relationship_score(row),
+                "strength": _plain_status_label(str(row.get("relationship_status", "") or "")),
+                "evidence": " | ".join(chips) if chips else "published source support",
+                "time-window overlap": _window_overlap_text(row),
+                "supporting rows": int(row.get("link_count", 0) or 0),
+                "stronger support": stronger_support,
+                "needs caution": caution_support,
+                "unresolved refs": int(row.get("unresolved_source_ref_count", 0) or 0),
+                "source_examples": ", ".join(_split_pipe_values(row.get("source_urls", ""), limit=2)),
             }
         )
-    return pd.DataFrame(rows)
+    ranked = pd.DataFrame(rows).sort_values(
+        ["overall score", "supporting rows", "stronger support", "counterparty / sector"],
+        ascending=[False, False, False, True],
+    ).reset_index(drop=True)
+    ranked.insert(0, "rank", range(1, len(ranked) + 1))
+    return ranked
+
+
+def _overview_summary_markdown(
+    ranked: pd.DataFrame,
+    *,
+    member_query: str,
+    family: str,
+    only_strong_links: bool,
+    top_n: int,
+) -> str:
+    if ranked.empty:
+        return "\n".join(
+            [
+                "### Overview",
+                "",
+                "No relationships match the current filters.",
+                "",
+                "Try a different House member, switch from sectors to funding recipients, or turn off the strong-links-only filter.",
+            ]
+        )
+    focus_names = [str(value) for value in ranked["member"].dropna().unique().tolist() if str(value).strip()]
+    focus_label = ", ".join(focus_names[:3])
+    lines = [
+        "### Overview",
+        "",
+        f"- Showing the top `{min(int(top_n), len(ranked))}` `{_plain_family_label(family).lower()}` for `{focus_label}`.",
+        f"- Filtered to stronger links only: `{str(bool(only_strong_links)).lower()}`.",
+        f"- Highest score in this view: `{int(ranked['overall score'].max())}`.",
+        "- Pick one relationship below to see the evidence breakdown and coarse evidence window.",
+    ]
+    if not str(member_query or "").strip():
+        lines.append("- Tip: search one House member for the clearest first read.")
+    return "\n".join(lines)
+
+
+def _relationship_options(ranked: pd.DataFrame) -> list[tuple[str, str]]:
+    if ranked.empty:
+        return []
+    options: list[tuple[str, str]] = []
+    for row in ranked.to_dict("records"):
+        label = f"{row['member']} -> {row['counterparty / sector']} (score {row['overall score']})"
+        options.append((label, str(row["relationship_id"])))
+    return options
+
+
+def _select_edge_row(edges: pd.DataFrame, relationship_id: str) -> Dict[str, Any] | None:
+    if edges.empty or not relationship_id:
+        return None
+    matched = edges[edges["edge_id"] == relationship_id]
+    if matched.empty:
+        return None
+    return matched.head(1).to_dict("records")[0]
+
+
+def _relationship_detail_markdown(edges: pd.DataFrame, relationship_id: str) -> str:
+    row = _select_edge_row(edges, relationship_id)
+    if not row:
+        return "Select a relationship to inspect why it appears in this released slice."
+    family = str(row.get("relationship_family", "") or "")
+    chips = _edge_evidence_chips(row)
+    reason_codes = [_plain_reason_code(item) for item in _split_pipe_values(row.get("reason_codes", ""), limit=8)]
+    urls = _split_pipe_values(row.get("source_urls", ""), limit=5)
+    lines = [
+        f"### {row.get('member_name') or row.get('member_slug')} -> {row.get('target_label')}",
+        "",
+        f"- Relationship view: `{_plain_family_label(family)}`",
+        f"- Strength label: `{_plain_status_label(str(row.get('relationship_status', '') or ''))}`",
+        f"- Overall score: `{_relationship_score(row)}`",
+        f"- Supporting relationship rows: `{int(row.get('link_count', 0) or 0)}`",
+        f"- Stronger-support rows: `{int(row.get('linked_count', 0) or 0) if family == 'recipient' else int(row.get('strong_event_count', 0) or 0)}`",
+        f"- Caution / weaker rows: `{int(row.get('review_count', 0) or 0) if family == 'recipient' else int(row.get('weak_event_count', 0) or 0)}`",
+        f"- Unresolved source refs still counted: `{int(row.get('unresolved_source_ref_count', 0) or 0)}`",
+        f"- Evidence signals: `{', '.join(chips) if chips else 'published source support'}`",
+        f"- Time-window overlap: `{_window_overlap_text(row)}`",
+    ]
+    if reason_codes:
+        lines.extend(["", "#### Why it is linked in this slice", ""])
+        lines.extend(f"- {item}" for item in reason_codes)
+    if urls:
+        lines.extend(["", "#### Example published source URLs", ""])
+        lines.extend(f"- {item}" for item in urls)
+    return "\n".join(lines)
+
+
+def _timeline_window_from_url(url: str) -> tuple[int, str, str]:
+    normalized = str(url or "").strip()
+    if not normalized:
+        return (99, "Published source", "No public URL attached in this row")
+    if "/ptr-pdfs/" in normalized or "/financial-pdfs/" in normalized:
+        match = re.search(r"/(\d{4})/", normalized)
+        year_label = match.group(1) if match else "Disclosure year"
+        kind = "Trade disclosure" if "/ptr-pdfs/" in normalized else "Annual disclosure"
+        return (10, year_label, kind)
+    if "BILLSTATUS-118" in normalized:
+        return (20, "2023-2024", "Bill and vote records (118th Congress)")
+    if "BILLSTATUS-119" in normalized:
+        return (30, "2025-2026", "Bill and vote records (119th Congress)")
+    if "usaspending.gov/award/" in normalized:
+        return (40, "Published award record", "Federal award record")
+    if "committee_info" in normalized:
+        return (50, "Current committee reference", "Committee roster")
+    return (60, "Published source", urlparse(normalized).netloc if normalized.startswith("http") else "Published source")
+
+
+def _relationship_timeline_html(edges: pd.DataFrame, relationship_id: str) -> str:
+    row = _select_edge_row(edges, relationship_id)
+    if not row:
+        return "<div style=\"padding: 1rem; border: 1px solid #d6d0c4; background: #fffdf8; color: #3a3a3a;\">Choose a relationship to see its evidence window.</div>"
+    entries: list[tuple[int, str, str, str]] = []
+    seen: set[tuple[str, str, str]] = set()
+    for url in _split_pipe_values(row.get("source_urls", ""), limit=8):
+        sort_key, window_label, track_label = _timeline_window_from_url(url)
+        detail = url
+        dedupe_key = (window_label, track_label, detail)
+        if dedupe_key not in seen:
+            seen.add(dedupe_key)
+            entries.append((sort_key, window_label, track_label, detail))
+    if int(row.get("profile_link_count", 0) or 0) > 0:
+        entries.append((70, "Undated support", "Member profile support", "Profile-based support is included in this relationship summary."))
+    if int(row.get("unresolved_source_ref_count", 0) or 0) > 0:
+        entries.append((80, "Partly unresolved", "Some official references remain unresolved", f"{int(row.get('unresolved_source_ref_count', 0) or 0)} unresolved refs are still counted in this released row."))
+    entries = sorted(entries, key=lambda item: (item[0], item[1], item[2], item[3]))
+    if not entries:
+        return "<div style=\"padding: 1rem; border: 1px solid #d6d0c4; background: #fffdf8; color: #3a3a3a;\">No evidence-window entries are available for this relationship.</div>"
+    cards = []
+    for _, window_label, track_label, detail in entries[:8]:
+        cards.append(
+            "<div style=\"display:flex; gap:16px; align-items:flex-start; margin:0 0 16px 0;\">"
+            f"<div style=\"min-width:120px; font-weight:700; color:#6b4e16;\">{html.escape(window_label)}</div>"
+            "<div style=\"border-left:3px solid #c08d2e; padding-left:14px;\">"
+            f"<div style=\"font-weight:700; color:#1f2b2d;\">{html.escape(track_label)}</div>"
+            f"<div style=\"color:#3d3d3d; margin-top:4px;\">{html.escape(detail)}</div>"
+            "</div>"
+            "</div>"
+        )
+    return (
+        "<div style=\"border:1px solid #d6d0c4; border-radius:12px; background:#fffdf8; padding:16px;\">"
+        "<div style=\"font-weight:700; margin-bottom:10px; color:#1f2b2d;\">Why this relationship appears</div>"
+        "<div style=\"color:#5c5c5c; margin-bottom:14px;\">This is a coarse evidence window based on the time hints published in this release. It is not exact chronology.</div>"
+        + "".join(cards)
+        + "</div>"
+    )
+
+
+def _graph_table(edges: pd.DataFrame) -> pd.DataFrame:
+    ranked = _rank_relationships(edges)
+    if ranked.empty:
+        return ranked
+    return ranked[
+        [
+            "rank",
+            "member",
+            "counterparty / sector",
+            "overall score",
+            "strength",
+            "evidence",
+            "time-window overlap",
+            "supporting rows",
+        ]
+    ]
 
 
 def _filter_events(events: pd.DataFrame, member_query: str, event_type: str, score_label: str, text_query: str) -> pd.DataFrame:
@@ -515,7 +779,6 @@ def _event_detail(events: pd.DataFrame, provenance: pd.DataFrame, event_id: str)
 def build_app(copy_path: str | Path):
     data = load_release_data(copy_path)
     events = data["events"]
-    links = data["links"]
     nodes = data["graph_nodes"]
     edges = data["graph_edges"]
     provenance = data["event_provenance"]
@@ -543,25 +806,97 @@ def build_app(copy_path: str | Path):
     event_id_choices = sorted(events["event_id"].dropna().unique().tolist())
     graph_defaults = data["graph_config"].get("default_filters") or {}
     overview_member_limit = int(graph_defaults.get("overview_member_limit", 8))
+    default_member_search = str(graph_defaults.get("default_member_search", "") or "")
 
     with gr.Blocks(title=copy_payload.get("title", "Congress Public Records Slice")) as app:
         gr.Markdown(copy_payload.get("welcome_markdown", copy_payload.get("landing_markdown", "")))
-        with gr.Tab("Network Graph"):
+        with gr.Tab("Overview"):
+            gr.Markdown(
+                "### Start here\n\n"
+                "Pick one House member, choose whether you want sectors or funding recipients, and read the ranked list first."
+            )
+            with gr.Row():
+                overview_member = gr.Textbox(label="House member", value=default_member_search)
+                overview_family = gr.Dropdown(label="Show", choices=[("Sectors", "sector"), ("Funding recipients", "recipient")], value="sector")
+                overview_only_strong = gr.Checkbox(label="Only strong links", value=True)
+                overview_top_n = gr.Slider(label="Show top relationships", minimum=5, maximum=40, step=5, value=10)
+            if example_member_choices:
+                gr.Examples(examples=example_member_choices, inputs=[overview_member], label="Try one of these example members")
+            overview_summary_md = gr.Markdown()
+            overview_df = gr.Dataframe(interactive=False)
+            relationship_choice = gr.Dropdown(label="Relationship to explain", choices=[], value=None)
+            overview_detail_md = gr.Markdown()
+            overview_timeline_html = gr.HTML()
+
+            def _overview_edges(member_query: str, family: str, only_strong: bool, top_n: int) -> pd.DataFrame:
+                return _filter_graph(
+                    edges,
+                    family,
+                    member_query,
+                    "",
+                    "all",
+                    "stronger" if only_strong else "all",
+                    True,
+                    top_n,
+                    overview_member_limit,
+                )
+
+            def _update_overview(member_query: str, family: str, only_strong: bool, top_n: int):
+                filtered_edges = _overview_edges(member_query, family, only_strong, top_n)
+                ranked = _rank_relationships(filtered_edges)
+                options = _relationship_options(ranked)
+                selected = options[0][1] if options else None
+                display = ranked.drop(columns=["relationship_id", "source_examples"], errors="ignore")
+                return (
+                    _overview_summary_markdown(
+                        ranked,
+                        member_query=member_query,
+                        family=family,
+                        only_strong_links=only_strong,
+                        top_n=top_n,
+                    ),
+                    display,
+                    gr.update(choices=options, value=selected),
+                    _relationship_detail_markdown(filtered_edges, selected or ""),
+                    _relationship_timeline_html(filtered_edges, selected or ""),
+                )
+
+            def _update_overview_detail(member_query: str, family: str, only_strong: bool, top_n: int, relationship_id: str):
+                filtered_edges = _overview_edges(member_query, family, only_strong, top_n)
+                return _relationship_detail_markdown(filtered_edges, relationship_id), _relationship_timeline_html(filtered_edges, relationship_id)
+
+            for control in (overview_member, overview_family, overview_only_strong, overview_top_n):
+                control.change(
+                    _update_overview,
+                    [overview_member, overview_family, overview_only_strong, overview_top_n],
+                    [overview_summary_md, overview_df, relationship_choice, overview_detail_md, overview_timeline_html],
+                )
+            relationship_choice.change(
+                _update_overview_detail,
+                [overview_member, overview_family, overview_only_strong, overview_top_n, relationship_choice],
+                [overview_detail_md, overview_timeline_html],
+            )
+            app.load(
+                _update_overview,
+                [overview_member, overview_family, overview_only_strong, overview_top_n],
+                [overview_summary_md, overview_df, relationship_choice, overview_detail_md, overview_timeline_html],
+            )
+        with gr.Tab("Explore Graph (optional)"):
             gr.Markdown(_graph_intro_markdown(data["graph_config"]))
             with gr.Row():
-                family = gr.Dropdown(label="Relationship view", choices=graph_family_choices, value=str(graph_defaults.get("relationship_family", "sector")))
-                member_graph_query = gr.Textbox(label="House member to focus", value=str(graph_defaults.get("default_member_search", "")))
+                family = gr.Dropdown(label="Show", choices=graph_family_choices, value=str(graph_defaults.get("relationship_family", "sector")))
+                member_graph_query = gr.Textbox(label="House member to focus", value=default_member_search)
                 target_query = gr.Textbox(label="Recipient or sector search")
                 graph_score = gr.Dropdown(label="Score label", choices=graph_score_choices, value="all")
-                review_status = gr.Dropdown(label="Relationship strength", choices=graph_status_choices, value=str(graph_defaults.get("review_status", "stronger")))
+                review_status = gr.Dropdown(label="Which links to show", choices=graph_status_choices, value=str(graph_defaults.get("review_status", "stronger")))
             if example_member_choices:
                 gr.Examples(examples=example_member_choices, inputs=[member_graph_query], label="Try one of these example members")
             with gr.Row():
-                hide_unresolved_only = gr.Checkbox(label="Hide unresolved relationships", value=bool(graph_defaults.get("hide_unresolved_only", True)))
-                max_edges = gr.Slider(label="Max visible relationships", minimum=25, maximum=300, step=25, value=int(graph_defaults.get("max_edges", 60)))
+                hide_unresolved_only = gr.Checkbox(label="Hide unresolved links", value=bool(graph_defaults.get("hide_unresolved_only", True)))
+                max_edges = gr.Slider(label="Show top relationships", minimum=25, maximum=300, step=25, value=int(graph_defaults.get("max_edges", 60)))
             graph_summary_md = gr.Markdown()
             graph_html = gr.HTML()
-            gr.Markdown("#### Relationships in this view")
+            gr.Markdown("#### Relationship list for this graph view")
             graph_df = gr.Dataframe(interactive=False)
             def _update_graph(family: str, member_graph_query: str, target_query: str, graph_score: str, review_status: str, hide_unresolved_only: bool, max_edges: int):
                 filtered_edges = _filter_graph(edges, family, member_graph_query, target_query, graph_score, review_status, hide_unresolved_only, max_edges, overview_member_limit)
@@ -578,7 +913,7 @@ def build_app(copy_path: str | Path):
             for control in (family, member_graph_query, target_query, graph_score, review_status, hide_unresolved_only, max_edges):
                 control.change(_update_graph, [family, member_graph_query, target_query, graph_score, review_status, hide_unresolved_only, max_edges], [graph_summary_md, graph_html, graph_df])
             app.load(_update_graph, [family, member_graph_query, target_query, graph_score, review_status, hide_unresolved_only, max_edges], [graph_summary_md, graph_html, graph_df])
-        with gr.Tab("Explore"):
+        with gr.Tab("Search Events"):
             with gr.Row():
                 member_query = gr.Textbox(label="Member name or slug")
                 event_type = gr.Dropdown(label="Event type", choices=event_type_choices, value="all")