ADD: test log

exp_pipeline/pipeline.py

@@ -15,6 +15,7 @@ def run_pipeline(split: str = "train"):
     data_loader = DataLoader()
     raw_data = data_loader.get_passage_dataset(split)
     logger.info(f"Loaded {len(raw_data)} samples from MS MARCO Passage Ranking [{split}]")
+    print("data_loader\n")
 
     # 2. 预处理数据
     preprocessor = Preprocessor()
@@ -22,6 +23,7 @@ def run_pipeline(split: str = "train"):
     if hasattr(raw_data, "to_dict"):
         raw_data = raw_data.to_dict()
         raw_data = [dict(zip(raw_data.keys(), v)) for v in zip(*raw_data.values())]
+    print("raw_data\n")
 
     # MS MARCO Passage v2.1: 用passages["passage_text"]字段
     passages = []
@@ -30,6 +32,7 @@ def run_pipeline(split: str = "train"):
             passages.extend(item["passages"]["passage_text"])
     processed = preprocessor.preprocess_passages(passages)
     texts = [p["text"] for p in processed]
+    print("texts\n")
 
     logger.info(f"Processed {len(texts)} passages")