GOLD PATH

src/envs.py

@@ -8,7 +8,15 @@ XAI_API_KEY = os.getenv("XAI_API_KEY")
 
 CORPUS_PATH = "data/5/corpus.jsonl"
 QUESTIONS_PATH = "data/5/questions_public.jsonl"
-GOLD_PATH = "secret/5/answers_gold.jsonl"
+# GOLD_PATH = "secret/5/answers_gold.jsonl"
+
+from huggingface_hub import hf_hub_download
+
+HF_TOKEN = os.getenv("HF_TOKEN")
+
+# Download from a dataset
+
+GOLD_PATH = hf_hub_download(repo_id="datakomarov/RAG-data", filename="answers_gold_5.jsonl", repo_type="dataset", token=HF_TOKEN, local_dir=".")
 
 def load_jsonl(path):
     return [json.loads(l) for l in open(path, encoding="utf-8")]