Add detailed logging to preprocessing to debug missing audio slices

pipeline/training.py

@@ -64,19 +64,49 @@ def preprocess(model_name: str, audio_path: str, sample_rate: int = 40000):
     ]
 
     logger.info(f"Running preprocessing for {model_name}...")
+    logger.info(f"Command: {' '.join(command)}")
+    logger.info(f"Dataset dir contents: {os.listdir(dataset_dir)}")
+
     result = subprocess.run(command, capture_output=True, text=True, cwd=APPLIO_DIR)
 
+    logger.info(f"Preprocess returncode: {result.returncode}")
+    if result.stdout:
+        logger.info(f"Preprocess stdout: {result.stdout[-1000:]}")
+    if result.stderr:
+        logger.info(f"Preprocess stderr: {result.stderr[-1000:]}")
+
     if result.returncode != 0:
-        logger.error(f"Preprocess stderr: {result.stderr}")
         raise RuntimeError(f"Preprocessing failed: {result.stderr[-500:]}")
 
+    # Log what was created in exp_dir
+    logger.info(f"Contents of exp_dir ({exp_dir}):")
+    for item in os.listdir(exp_dir):
+        full = os.path.join(exp_dir, item)
+        if os.path.isdir(full):
+            contents = os.listdir(full)
+            logger.info(f"  {item}/ ({len(contents)} files): {contents[:5]}")
+        else:
+            logger.info(f"  {item} ({os.path.getsize(full)} bytes)")
+
+    # Check multiple possible output directories
     sliced_dir = os.path.join(exp_dir, "sliced_audios")
-    if not os.path.exists(sliced_dir) or len(os.listdir(sliced_dir)) == 0:
-        raise RuntimeError("Preprocessing produced no audio slices. Check your input audio.")
-
-    n_slices = len(os.listdir(sliced_dir))
-    logger.info(f"Preprocessing complete: {n_slices} slices created.")
-    return n_slices
+    sliced_16k_dir = os.path.join(exp_dir, "sliced_audios_16k")
+
+    if os.path.exists(sliced_dir) and len(os.listdir(sliced_dir)) > 0:
+        n_slices = len(os.listdir(sliced_dir))
+        logger.info(f"Preprocessing complete: {n_slices} slices created.")
+        return n_slices
+    elif os.path.exists(sliced_16k_dir) and len(os.listdir(sliced_16k_dir)) > 0:
+        n_slices = len(os.listdir(sliced_16k_dir))
+        logger.info(f"Preprocessing complete: {n_slices} 16k slices created (no native rate slices).")
+        return n_slices
+    else:
+        raise RuntimeError(
+            f"Preprocessing produced no audio slices. "
+            f"exp_dir contents: {os.listdir(exp_dir)}. "
+            f"stdout: {result.stdout[-300:] if result.stdout else 'empty'}. "
+            f"stderr: {result.stderr[-300:] if result.stderr else 'empty'}"
+        )
 
 
 @spaces.GPU(duration=120)