Upload 20 files
Browse files- requirements.txt +1 -0
- routers/backup.py +12 -15
requirements.txt
CHANGED
|
@@ -4,4 +4,5 @@ websockets==14.1
|
|
| 4 |
python-multipart==0.0.18
|
| 5 |
aiofiles==24.1.0
|
| 6 |
httpx==0.28.1
|
|
|
|
| 7 |
|
|
|
|
| 4 |
python-multipart==0.0.18
|
| 5 |
aiofiles==24.1.0
|
| 6 |
httpx==0.28.1
|
| 7 |
+
huggingface_hub>=0.25.0
|
| 8 |
|
routers/backup.py
CHANGED
|
@@ -122,20 +122,18 @@ async def list_backups(request: Request):
|
|
| 122 |
|
| 123 |
|
| 124 |
def _upload_to_hf(creds: dict, zone_name: str, archive_path: Path):
|
| 125 |
-
"""Upload archive directly to HuggingFace Dataset
|
|
|
|
|
|
|
| 126 |
file_path = f"{creds['path_prefix']}/{zone_name}.tar.gz"
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
content=f.read(),
|
| 136 |
-
)
|
| 137 |
-
if resp.status_code not in (200, 201):
|
| 138 |
-
raise ValueError(f"HF upload error: {resp.status_code} {resp.text}")
|
| 139 |
|
| 140 |
|
| 141 |
@router.post("/zone/{zone_name}")
|
|
@@ -236,11 +234,10 @@ async def backup_all(request: Request, background_tasks: BackgroundTasks):
|
|
| 236 |
def _download_from_hf(creds: dict, zone_name: str) -> bytes:
|
| 237 |
"""Download archive directly from HuggingFace Dataset."""
|
| 238 |
file_path = f"{creds['path_prefix']}/{zone_name}.tar.gz"
|
| 239 |
-
with httpx.Client(timeout=300) as client:
|
| 240 |
resp = client.get(
|
| 241 |
f"https://huggingface.co/datasets/{creds['repo']}/resolve/main/{file_path}",
|
| 242 |
headers={"Authorization": f"Bearer {creds['hf_token']}"},
|
| 243 |
-
follow_redirects=True,
|
| 244 |
)
|
| 245 |
if resp.status_code == 404:
|
| 246 |
raise FileNotFoundError(f"Backup zone '{zone_name}' khong ton tai")
|
|
|
|
| 122 |
|
| 123 |
|
| 124 |
def _upload_to_hf(creds: dict, zone_name: str, archive_path: Path):
|
| 125 |
+
"""Upload archive directly to HuggingFace Dataset via huggingface_hub."""
|
| 126 |
+
from huggingface_hub import HfApi
|
| 127 |
+
|
| 128 |
file_path = f"{creds['path_prefix']}/{zone_name}.tar.gz"
|
| 129 |
+
api = HfApi(token=creds["hf_token"])
|
| 130 |
+
api.upload_file(
|
| 131 |
+
path_or_fileobj=str(archive_path),
|
| 132 |
+
path_in_repo=file_path,
|
| 133 |
+
repo_id=creds["repo"],
|
| 134 |
+
repo_type="dataset",
|
| 135 |
+
commit_message=f"Backup zone: {zone_name}",
|
| 136 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 137 |
|
| 138 |
|
| 139 |
@router.post("/zone/{zone_name}")
|
|
|
|
| 234 |
def _download_from_hf(creds: dict, zone_name: str) -> bytes:
|
| 235 |
"""Download archive directly from HuggingFace Dataset."""
|
| 236 |
file_path = f"{creds['path_prefix']}/{zone_name}.tar.gz"
|
| 237 |
+
with httpx.Client(timeout=300, follow_redirects=True) as client:
|
| 238 |
resp = client.get(
|
| 239 |
f"https://huggingface.co/datasets/{creds['repo']}/resolve/main/{file_path}",
|
| 240 |
headers={"Authorization": f"Bearer {creds['hf_token']}"},
|
|
|
|
| 241 |
)
|
| 242 |
if resp.status_code == 404:
|
| 243 |
raise FileNotFoundError(f"Backup zone '{zone_name}' khong ton tai")
|