kokokoasd commited on
Commit
5c4bd29
·
verified ·
1 Parent(s): cbc776e

Upload 20 files

Browse files
Files changed (2) hide show
  1. requirements.txt +1 -0
  2. routers/backup.py +12 -15
requirements.txt CHANGED
@@ -4,4 +4,5 @@ websockets==14.1
4
  python-multipart==0.0.18
5
  aiofiles==24.1.0
6
  httpx==0.28.1
 
7
 
 
4
  python-multipart==0.0.18
5
  aiofiles==24.1.0
6
  httpx==0.28.1
7
+ huggingface_hub>=0.25.0
8
 
routers/backup.py CHANGED
@@ -122,20 +122,18 @@ async def list_backups(request: Request):
122
 
123
 
124
  def _upload_to_hf(creds: dict, zone_name: str, archive_path: Path):
125
- """Upload archive directly to HuggingFace Dataset API."""
 
 
126
  file_path = f"{creds['path_prefix']}/{zone_name}.tar.gz"
127
- hf_token = creds["hf_token"]
128
- repo = creds["repo"]
129
-
130
- with httpx.Client(timeout=300) as client:
131
- with open(archive_path, "rb") as f:
132
- resp = client.post(
133
- f"{HF_API}/datasets/{repo}/upload/main/{file_path}",
134
- headers={"Authorization": f"Bearer {hf_token}"},
135
- content=f.read(),
136
- )
137
- if resp.status_code not in (200, 201):
138
- raise ValueError(f"HF upload error: {resp.status_code} {resp.text}")
139
 
140
 
141
  @router.post("/zone/{zone_name}")
@@ -236,11 +234,10 @@ async def backup_all(request: Request, background_tasks: BackgroundTasks):
236
  def _download_from_hf(creds: dict, zone_name: str) -> bytes:
237
  """Download archive directly from HuggingFace Dataset."""
238
  file_path = f"{creds['path_prefix']}/{zone_name}.tar.gz"
239
- with httpx.Client(timeout=300) as client:
240
  resp = client.get(
241
  f"https://huggingface.co/datasets/{creds['repo']}/resolve/main/{file_path}",
242
  headers={"Authorization": f"Bearer {creds['hf_token']}"},
243
- follow_redirects=True,
244
  )
245
  if resp.status_code == 404:
246
  raise FileNotFoundError(f"Backup zone '{zone_name}' khong ton tai")
 
122
 
123
 
124
  def _upload_to_hf(creds: dict, zone_name: str, archive_path: Path):
125
+ """Upload archive directly to HuggingFace Dataset via huggingface_hub."""
126
+ from huggingface_hub import HfApi
127
+
128
  file_path = f"{creds['path_prefix']}/{zone_name}.tar.gz"
129
+ api = HfApi(token=creds["hf_token"])
130
+ api.upload_file(
131
+ path_or_fileobj=str(archive_path),
132
+ path_in_repo=file_path,
133
+ repo_id=creds["repo"],
134
+ repo_type="dataset",
135
+ commit_message=f"Backup zone: {zone_name}",
136
+ )
 
 
 
 
137
 
138
 
139
  @router.post("/zone/{zone_name}")
 
234
  def _download_from_hf(creds: dict, zone_name: str) -> bytes:
235
  """Download archive directly from HuggingFace Dataset."""
236
  file_path = f"{creds['path_prefix']}/{zone_name}.tar.gz"
237
+ with httpx.Client(timeout=300, follow_redirects=True) as client:
238
  resp = client.get(
239
  f"https://huggingface.co/datasets/{creds['repo']}/resolve/main/{file_path}",
240
  headers={"Authorization": f"Bearer {creds['hf_token']}"},
 
241
  )
242
  if resp.status_code == 404:
243
  raise FileNotFoundError(f"Backup zone '{zone_name}' khong ton tai")