Spaces:
Running
Running
File size: 1,483 Bytes
1bf5b23 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 | """Tests for evaluator.score_submission().
Run from the EgoMemReason-Space/ directory:
python -m pytest tests/ -q
"""
import json
import os
import pathlib
import sys
import pytest
ROOT = pathlib.Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
import evaluator
ANN = ROOT / "annotations_private.json"
ORACLE = ROOT / "tests" / "fixtures" / "oracle_submission.json"
ALL_A = ROOT / "tests" / "fixtures" / "all_a_submission.json"
pytestmark = pytest.mark.skipif(
not ANN.exists(),
reason=f"{ANN} not present (copy from ../EgoMemReason-EvalAI.archived/)",
)
def test_oracle_scores_100():
metrics = evaluator.score_submission(str(ORACLE), str(ANN))
for k, v in metrics.items():
assert v == 100.0, f"{k} should be 100.0, got {v}"
def test_all_a_scores_around_14():
# All-A's exact score depends on the A-letter frequency in the dataset
# — we measured 14.2% during the EvalAI port. Allow a wide band.
metrics = evaluator.score_submission(str(ALL_A), str(ANN))
assert 10.0 <= metrics["Overall"] <= 20.0, metrics
def test_broken_submission_raises(tmp_path):
broken = tmp_path / "broken.json"
json.dump(
[{"example_id": 1, "predicted_answer": "ZZ"}], # bogus letter + only 1 row
broken.open("w"),
)
with pytest.raises(ValueError) as exc:
evaluator.score_submission(str(broken), str(ANN))
assert "must be one of" in str(exc.value)
assert "missing" in str(exc.value)
|