claudeattempt dataset

scripts/train.py

@@ -11,6 +11,19 @@ from datasets import load_dataset
 from myolmoe import MyOlmoeForCausalLM, OlmoeConfig
 import os
 
+import datasets
+
+# Clear the cache
+datasets.disable_caching()
+
+# Or manually clear cache
+import shutil
+import os
+cache_dir = os.path.expanduser("~/.cache/huggingface/datasets")
+if os.path.exists(cache_dir):
+    shutil.rmtree(cache_dir)
+
+
 def main():
     # Load config - first try from local file, then from pretrained
     config_path = os.path.join("myolmoe", "config.json")
@@ -33,10 +46,8 @@ def main():
     tokenizer.pad_token = tokenizer.eos_token
     
     # Load dataset
-    os.environ["HF_DATASETS_CACHE"] = "/tmp/hf-datasets-cache"  # or any valid path
-    dataset = load_dataset("allenai/tulu-v2-sft-mixture", split="train", download_mode="force_redownload")
-
-
+    dataset = load_dataset("allenai/tulu-v2-sft-mixture", split="train")
+    
     def tokenize_function(examples):
         return tokenizer(
             examples["text"],