src/submission/check_validity.py

import json

def check_submission(path):
    lines = open(path, encoding="utf-8").read().splitlines()
    for i, line in enumerate(lines):
        try:
            item = json.loads(line)
        except:
            return False, f"Line {i} is not valid JSON"

        if "id" not in item or "answer" not in item:
            return False, f"Missing id or answer in line {i}"

        if "doc_ids" in item:
            if not isinstance(item["doc_ids"], list):
                return False, f"doc_ids must be list in line {i}"
            if len(item["doc_ids"]) > 10:
                return False, f"Maximum 10 doc_ids allowed (line {i})"

    return True, f"{len(lines)} items validated"