Spaces:
Running
Running
File size: 3,271 Bytes
03815d6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 | """Tests for Mode C evaluation pipeline + stats utilities."""
from __future__ import annotations
import pytest
from eval.bootstrap_ci import bootstrap_ci, cohens_d, permutation_test
from eval.mode_c_real_cases import (
DEFAULT_DATASET,
ScriptedAnalyzerAdapter,
aggregate,
load_dataset,
per_category_breakdown,
run_eval,
)
@pytest.mark.unit
def test_dataset_loads_and_is_expected_size():
data = load_dataset(DEFAULT_DATASET)
# Lower bound — may grow over time as community contributes
assert len(data) >= 130
@pytest.mark.unit
def test_dataset_has_scam_and_benign():
data = load_dataset(DEFAULT_DATASET)
scam = sum(1 for s in data if s["ground_truth"]["is_scam"])
benign = sum(1 for s in data if not s["ground_truth"]["is_scam"])
assert scam > 0
assert benign > 0
assert scam + benign == len(data)
@pytest.mark.unit
def test_dataset_has_novel_subset_for_temporal_eval():
"""Temporal generalization eval requires n>=30 for statistical power."""
data = load_dataset(DEFAULT_DATASET)
novel = [s for s in data if s["ground_truth"]["difficulty"] == "novel"]
assert len(novel) >= 30, f"Only {len(novel)} novel scenarios — need 30+ for CI"
@pytest.mark.unit
def test_bootstrap_ci_reasonable():
# 80% positive rate in a large sample → CI should bracket 0.8
samples = [1.0] * 80 + [0.0] * 20
point, lo, hi = bootstrap_ci(samples, n_resamples=1000, seed=42)
assert abs(point - 0.8) < 0.001
assert lo < 0.8 < hi
assert (hi - lo) < 0.2 # reasonably tight CI
@pytest.mark.unit
def test_permutation_test_detects_difference():
group_a = [1.0] * 50
group_b = [0.0] * 50
p = permutation_test(group_a, group_b, n_permutations=1000, seed=42)
assert p < 0.05
@pytest.mark.unit
def test_permutation_test_no_difference():
group_a = [0.5, 0.5, 0.5, 0.5]
group_b = [0.5, 0.5, 0.5, 0.5]
p = permutation_test(group_a, group_b, n_permutations=500, seed=42)
assert p > 0.05
@pytest.mark.unit
def test_cohens_d_large_effect():
group_a = [10.0, 11.0, 9.0, 10.5, 11.5, 9.5]
group_b = [1.0, 2.0, 0.5, 1.5, 2.5, 0.0]
d = cohens_d(group_a, group_b)
assert d > 0.8 # large effect
@pytest.mark.integration
def test_scripted_analyzer_catches_majority_of_scams():
data = load_dataset(DEFAULT_DATASET)
analyzer = ScriptedAnalyzerAdapter()
results = run_eval(analyzer, data, threshold=0.5)
metrics = aggregate(results)
# Baseline scripted analyzer must catch at least 40% of scams (sanity bound)
assert metrics.detection_rate >= 0.40
# Benign cases should have SOME false positives (rule-based is crude)
# but should NOT be > 60% (otherwise the analyzer is useless)
assert metrics.false_positive_rate <= 0.60
@pytest.mark.integration
def test_per_category_covers_all_scam_categories():
data = load_dataset(DEFAULT_DATASET)
analyzer = ScriptedAnalyzerAdapter()
results = run_eval(analyzer, data)
by_cat = per_category_breakdown(results)
for required_cat in (
"otp_theft",
"kyc_fraud",
"loan_app_fraud",
"investment_fraud",
"impersonation",
"benign",
):
assert required_cat in by_cat, f"Missing category: {required_cat}"
|