Delete graders.py with huggingface_hub

graders.py

@@ -1,178 +0,0 @@
-"""Deterministic graders and reward helpers for SupportDesk."""
-
-from __future__ import annotations
-
-import re
-from dataclasses import dataclass
-
-from models import SupportCaseProgress
-from tasks import SupportTaskSpec, get_task
-
-STRICT_SCORE_EPSILON = 0.01
-
-
-@dataclass(frozen=True)
-class GradeBreakdown:
-    """A scored view of how close a case is to the gold solution."""
-
-    total_score: float
-    queue_score: float
-    priority_score: float
-    issue_type_score: float
-    requested_fields_score: float
-    reply_score: float
-    note_score: float
-    status_score: float
-    resolution_score: float
-    completed_milestones: tuple[str, ...]
-
-
-def _normalize(text: str | None) -> str:
-    if not text:
-        return ""
-    normalized = text.lower().replace("-", " ")
-    return re.sub(r"[^a-z0-9\s]", " ", normalized)
-
-
-def _marker_group_score(text: str | None, marker_groups: tuple[tuple[str, ...], ...]) -> float:
-    if not marker_groups:
-        return 1.0
-
-    normalized = _normalize(text)
-    if not normalized:
-        return 0.0
-
-    matches = 0
-    for group in marker_groups:
-        if any(_normalize(marker) in normalized for marker in group):
-            matches += 1
-    return matches / len(marker_groups)
-
-
-def _requested_fields_score(case: SupportCaseProgress, task: SupportTaskSpec) -> float:
-    required = set(task.required_requested_fields)
-    requested = set(case.requested_fields)
-
-    if not required:
-        return 1.0 if not requested else 0.0
-    if not requested:
-        return 0.0
-
-    matched = len(required.intersection(requested))
-    extras = len(requested.difference(required))
-    raw = matched / len(required)
-    penalty = min(0.25, extras * 0.05)
-    return max(0.0, raw - penalty)
-
-
-def _reply_penalty(case: SupportCaseProgress, task: SupportTaskSpec) -> float:
-    text = _normalize(case.reply)
-    if not text:
-        return 0.0
-    return 0.0 if not any(_normalize(marker) in text for marker in task.forbidden_reply_markers) else 0.5
-
-
-def _strict_open_unit_interval(score: float) -> float:
-    """Keep final task scores strictly within (0, 1) for evaluator compatibility."""
-
-    return min(1.0 - STRICT_SCORE_EPSILON, max(STRICT_SCORE_EPSILON, score))
-
-
-def grade_case(task: SupportTaskSpec, case: SupportCaseProgress) -> GradeBreakdown:
-    """Score a case deterministically with total_score strictly inside (0, 1)."""
-
-    queue_score = 1.0 if case.queue == task.gold_queue else 0.0
-    priority_score = 1.0 if case.priority == task.gold_priority else 0.0
-    issue_type_score = 1.0 if case.issue_type == task.gold_issue_type else 0.0
-    requested_fields_score = _requested_fields_score(case, task)
-    reply_score = max(0.0, _marker_group_score(case.reply, task.required_reply_markers) - _reply_penalty(case, task))
-    note_score = _marker_group_score(case.internal_note, task.required_note_markers)
-    status_score = 1.0 if case.status == task.gold_status else 0.0
-    resolution_score = 1.0 if case.resolution_code == task.gold_resolution_code else 0.0
-
-    weighted_total = (
-        queue_score * 0.15
-        + priority_score * 0.10
-        + issue_type_score * 0.10
-        + requested_fields_score * 0.15
-        + reply_score * 0.25
-        + note_score * 0.10
-        + status_score * 0.10
-        + resolution_score * 0.05
-    )
-
-    milestones: list[str] = []
-    if queue_score:
-        milestones.append("queue")
-    if priority_score:
-        milestones.append("priority")
-    if issue_type_score:
-        milestones.append("issue_type")
-    if requested_fields_score >= 0.99:
-        milestones.append("requested_fields")
-    if reply_score >= 0.99:
-        milestones.append("reply")
-    if note_score >= 0.99:
-        milestones.append("internal_note")
-    if status_score:
-        milestones.append("status")
-    if resolution_score:
-        milestones.append("resolution_code")
-
-    return GradeBreakdown(
-        total_score=round(_strict_open_unit_interval(weighted_total), 4),
-        queue_score=queue_score,
-        priority_score=priority_score,
-        issue_type_score=issue_type_score,
-        requested_fields_score=round(requested_fields_score, 4),
-        reply_score=round(reply_score, 4),
-        note_score=round(note_score, 4),
-        status_score=status_score,
-        resolution_score=resolution_score,
-        completed_milestones=tuple(milestones),
-    )
-
-
-def grade_task_id(task_id: str, case: SupportCaseProgress) -> GradeBreakdown:
-    """Convenience wrapper used by tests and evaluation scripts."""
-
-    return grade_case(get_task(task_id), case)
-
-
-class _TaskSpecificGrader:
-    """Importable task-specific grader wrapper for validator task discovery."""
-
-    task_id: str = ""
-
-    def grade(self, case: SupportCaseProgress) -> float:
-        return grade_task_id(self.task_id, case).total_score
-
-    def __call__(self, case: SupportCaseProgress) -> float:
-        return self.grade(case)
-
-
-class BillingRefundEasyGrader(_TaskSpecificGrader):
-    task_id = "billing_refund_easy"
-
-
-class AccountTakeoverMediumGrader(_TaskSpecificGrader):
-    task_id = "account_takeover_medium"
-
-
-class ApiIncidentHardGrader(_TaskSpecificGrader):
-    task_id = "api_incident_hard"
-
-
-class RegulatedExportExceptionHardGrader(_TaskSpecificGrader):
-    task_id = "regulated_export_exception_hard"
-
-
-__all__ = [
-    "AccountTakeoverMediumGrader",
-    "ApiIncidentHardGrader",
-    "BillingRefundEasyGrader",
-    "GradeBreakdown",
-    "RegulatedExportExceptionHardGrader",
-    "grade_case",
-    "grade_task_id",
-]