Add phd_research_os_v2/layer6/evaluator.py

phd_research_os_v2/layer6/evaluator.py

@@ -0,0 +1,165 @@
+"""
+Layer 6: Evaluation Harness
+==============================
+Regression gate, golden dataset management, quality metrics.
+"""
+
+import json
+import os
+from pathlib import Path
+from ..core.database import get_db, gen_id, now_iso, from_fixed
+
+
+REGRESSION_THRESHOLDS = {
+    "min_extraction_recall": 0.70,
+    "max_hallucination_rate": 0.10,
+    "min_epistemic_accuracy": 0.60,
+    "min_qualifier_preservation": 0.50,
+    "min_null_detection_rate": 0.30,
+}
+
+
+class Evaluator:
+    """Evaluation harness with regression gate and quality metrics."""
+    
+    def __init__(self, db_path: str = None, golden_path: str = "config/golden_dataset"):
+        self.db_path = db_path
+        self.golden_path = Path(golden_path)
+    
+    def compute_system_metrics(self) -> dict:
+        """Compute current system-wide quality metrics."""
+        conn = get_db(self.db_path)
+        
+        total = conn.execute("SELECT COUNT(*) FROM claims").fetchone()[0]
+        if total == 0:
+            conn.close()
+            return {"total_claims": 0, "message": "No claims to evaluate"}
+        
+        # Epistemic tag distribution
+        tags = conn.execute(
+            "SELECT epistemic_tag, COUNT(*) FROM claims GROUP BY epistemic_tag"
+        ).fetchall()
+        tag_dist = {dict(t)["epistemic_tag"]: list(dict(t).values())[1] for t in tags}
+        
+        # Status distribution
+        statuses = conn.execute(
+            "SELECT status, COUNT(*) FROM claims GROUP BY status"
+        ).fetchall()
+        status_dist = {dict(s)["status"]: list(dict(s).values())[1] for s in statuses}
+        
+        # Null result count
+        null_count = conn.execute(
+            "SELECT COUNT(*) FROM claims WHERE is_null_result = 1"
+        ).fetchone()[0]
+        
+        # Average confidence scores
+        avg_composite = conn.execute(
+            "SELECT AVG(composite_confidence) FROM claims WHERE composite_confidence IS NOT NULL"
+        ).fetchone()[0]
+        avg_evidence = conn.execute(
+            "SELECT AVG(evidence_quality) FROM claims WHERE evidence_quality IS NOT NULL"
+        ).fetchone()[0]
+        
+        # Section distribution
+        sections = conn.execute(
+            "SELECT source_section, COUNT(*) FROM claims WHERE source_section IS NOT NULL GROUP BY source_section"
+        ).fetchall()
+        section_dist = {dict(s)["source_section"]: list(dict(s).values())[1] for s in sections}
+        
+        # Qualifier stats
+        with_qualifiers = conn.execute(
+            "SELECT COUNT(*) FROM claims WHERE qualifiers IS NOT NULL AND qualifiers != '[]'"
+        ).fetchone()[0]
+        
+        # Canonical dedup ratio
+        canonical_count = conn.execute("SELECT COUNT(*) FROM canonical_claims").fetchone()[0]
+        dedup_ratio = canonical_count / total if total > 0 else 0
+        
+        conn.close()
+        
+        return {
+            "total_claims": total,
+            "epistemic_distribution": tag_dist,
+            "status_distribution": status_dist,
+            "null_results": null_count,
+            "null_rate": round(null_count / total, 3) if total > 0 else 0,
+            "avg_composite_confidence": round(from_fixed(int(avg_composite or 0)), 3),
+            "avg_evidence_quality": round(from_fixed(int(avg_evidence or 0)), 3),
+            "section_distribution": section_dist,
+            "claims_with_qualifiers": with_qualifiers,
+            "qualifier_rate": round(with_qualifiers / total, 3) if total > 0 else 0,
+            "canonical_claims": canonical_count,
+            "dedup_ratio": round(dedup_ratio, 3),
+        }
+    
+    def run_regression_gate(self, metrics: dict = None) -> dict:
+        """
+        Run regression gate against thresholds.
+        Returns pass/fail with details.
+        """
+        if metrics is None:
+            metrics = self.compute_system_metrics()
+        
+        if metrics.get("total_claims", 0) == 0:
+            return {"passed": False, "reason": "No claims to evaluate", "checks": []}
+        
+        checks = []
+        all_passed = True
+        
+        # Check qualifier preservation
+        qual_rate = metrics.get("qualifier_rate", 0)
+        qual_pass = qual_rate >= REGRESSION_THRESHOLDS["min_qualifier_preservation"]
+        checks.append({
+            "name": "Qualifier preservation",
+            "value": qual_rate,
+            "threshold": REGRESSION_THRESHOLDS["min_qualifier_preservation"],
+            "passed": qual_pass,
+        })
+        if not qual_pass:
+            all_passed = False
+        
+        # Check null detection
+        null_rate = metrics.get("null_rate", 0)
+        null_pass = null_rate >= REGRESSION_THRESHOLDS["min_null_detection_rate"] or metrics["total_claims"] < 50
+        checks.append({
+            "name": "Null result detection",
+            "value": null_rate,
+            "threshold": REGRESSION_THRESHOLDS["min_null_detection_rate"],
+            "passed": null_pass,
+            "note": "Skipped (< 50 claims)" if metrics["total_claims"] < 50 else None,
+        })
+        
+        # Check epistemic diversity (should have at least 2 distinct tags)
+        tag_count = len(metrics.get("epistemic_distribution", {}))
+        diversity_pass = tag_count >= 2
+        checks.append({
+            "name": "Epistemic diversity",
+            "value": tag_count,
+            "threshold": 2,
+            "passed": diversity_pass,
+        })
+        if not diversity_pass:
+            all_passed = False
+        
+        # Log the eval run
+        conn = get_db(self.db_path)
+        run_id = gen_id("EVAL")
+        conn.execute("""
+            INSERT INTO eval_runs (run_id, run_type, metrics, passed, pipeline_version, created_at)
+            VALUES (?, 'regression', ?, ?, '2.1.0', ?)
+        """, (run_id, json.dumps({"metrics": metrics, "checks": checks}),
+              int(all_passed), now_iso()))
+        conn.commit()
+        conn.close()
+        
+        return {
+            "passed": all_passed,
+            "run_id": run_id,
+            "checks": checks,
+            "metrics_summary": {
+                "total_claims": metrics["total_claims"],
+                "avg_confidence": metrics.get("avg_composite_confidence", 0),
+                "qualifier_rate": qual_rate,
+                "null_rate": null_rate,
+            }
+        }