slslslrhfem commited on
Commit ·
cb998ea
1
Parent(s): b4eefc1
change download mechanism
Browse files
app.py
CHANGED
|
@@ -22,7 +22,12 @@ def install_madmom():
|
|
| 22 |
|
| 23 |
install_madmom()
|
| 24 |
|
|
|
|
|
|
|
|
|
|
| 25 |
|
|
|
|
|
|
|
| 26 |
|
| 27 |
def download_data_from_hub():
|
| 28 |
base_dir = Path(".")
|
|
@@ -48,10 +53,14 @@ def download_data_from_hub():
|
|
| 48 |
if not all_folders_exist:
|
| 49 |
print(f"Downloading data folders from dataset: {data_repo_id}")
|
| 50 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 51 |
downloaded_path = snapshot_download(
|
| 52 |
repo_id=data_repo_id,
|
| 53 |
repo_type="dataset",
|
| 54 |
-
local_dir=str(
|
| 55 |
local_dir_use_symlinks=False,
|
| 56 |
token=token,
|
| 57 |
ignore_patterns=["*.md", "*.txt", ".gitattributes", "README.md"]
|
|
@@ -59,15 +68,33 @@ def download_data_from_hub():
|
|
| 59 |
|
| 60 |
print("Dataset downloaded successfully")
|
| 61 |
|
|
|
|
|
|
|
|
|
|
| 62 |
for folder_name in folders_to_check:
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
print(f"{folder_name}: {file_count:,} files")
|
| 67 |
-
downloaded_folders[folder_name] = str(
|
| 68 |
else:
|
| 69 |
-
print(f"{folder_name}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
downloaded_folders[folder_name] = None
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
else:
|
| 72 |
print("Data folders already exist locally")
|
| 73 |
for folder_name in folders_to_check:
|
|
@@ -329,8 +356,7 @@ h1 {
|
|
| 329 |
print("Starting Music Plagiarism Detection App...")
|
| 330 |
print("Downloading covers80 and ml_models folders...")
|
| 331 |
folders = download_data_from_hub()
|
| 332 |
-
|
| 333 |
-
from inference import inference
|
| 334 |
if folders.get("covers80") and folders.get("ml_models"):
|
| 335 |
print("All required data folders are ready")
|
| 336 |
elif folders.get("covers80") or folders.get("ml_models"):
|
|
|
|
| 22 |
|
| 23 |
install_madmom()
|
| 24 |
|
| 25 |
+
# Add current directory to Python path for ml_models
|
| 26 |
+
sys.path.insert(0, '.')
|
| 27 |
+
sys.path.insert(0, './ml_models')
|
| 28 |
|
| 29 |
+
# Import inference - any error will bubble up
|
| 30 |
+
from inference import inference
|
| 31 |
|
| 32 |
def download_data_from_hub():
|
| 33 |
base_dir = Path(".")
|
|
|
|
| 53 |
if not all_folders_exist:
|
| 54 |
print(f"Downloading data folders from dataset: {data_repo_id}")
|
| 55 |
|
| 56 |
+
# Download to a temporary directory first
|
| 57 |
+
temp_dir = base_dir / "temp_download"
|
| 58 |
+
temp_dir.mkdir(exist_ok=True)
|
| 59 |
+
|
| 60 |
downloaded_path = snapshot_download(
|
| 61 |
repo_id=data_repo_id,
|
| 62 |
repo_type="dataset",
|
| 63 |
+
local_dir=str(temp_dir),
|
| 64 |
local_dir_use_symlinks=False,
|
| 65 |
token=token,
|
| 66 |
ignore_patterns=["*.md", "*.txt", ".gitattributes", "README.md"]
|
|
|
|
| 68 |
|
| 69 |
print("Dataset downloaded successfully")
|
| 70 |
|
| 71 |
+
# Move folders from temp_download to current directory
|
| 72 |
+
import shutil
|
| 73 |
+
|
| 74 |
for folder_name in folders_to_check:
|
| 75 |
+
temp_folder_path = temp_dir / folder_name
|
| 76 |
+
target_folder_path = base_dir / folder_name
|
| 77 |
+
|
| 78 |
+
if temp_folder_path.exists():
|
| 79 |
+
if target_folder_path.exists():
|
| 80 |
+
shutil.rmtree(target_folder_path)
|
| 81 |
+
shutil.move(str(temp_folder_path), str(target_folder_path))
|
| 82 |
+
print(f"Moved {folder_name} to current directory")
|
| 83 |
+
|
| 84 |
+
file_count = len([f for f in target_folder_path.rglob("*") if f.is_file()])
|
| 85 |
print(f"{folder_name}: {file_count:,} files")
|
| 86 |
+
downloaded_folders[folder_name] = str(target_folder_path)
|
| 87 |
else:
|
| 88 |
+
print(f"{folder_name} not found in downloaded data")
|
| 89 |
+
# Check what's actually in temp directory
|
| 90 |
+
print(f"Contents of temp directory:")
|
| 91 |
+
for item in temp_dir.iterdir():
|
| 92 |
+
print(f" {item.name}")
|
| 93 |
downloaded_folders[folder_name] = None
|
| 94 |
+
|
| 95 |
+
# Clean up temp directory
|
| 96 |
+
shutil.rmtree(temp_dir)
|
| 97 |
+
|
| 98 |
else:
|
| 99 |
print("Data folders already exist locally")
|
| 100 |
for folder_name in folders_to_check:
|
|
|
|
| 356 |
print("Starting Music Plagiarism Detection App...")
|
| 357 |
print("Downloading covers80 and ml_models folders...")
|
| 358 |
folders = download_data_from_hub()
|
| 359 |
+
|
|
|
|
| 360 |
if folders.get("covers80") and folders.get("ml_models"):
|
| 361 |
print("All required data folders are ready")
|
| 362 |
elif folders.get("covers80") or folders.get("ml_models"):
|