Upload 6 files

README.md

@@ -1,39 +1,33 @@
----
-title: Sub2API HF Demo
-emoji: 🚀
-colorFrom: indigo
-colorTo: blue
-sdk: docker
-app_port: 8080
-pinned: false
----
-
-# Sub2API on Hugging Face Spaces (Demo / Experimental)
-
-> 这是一个 **演示性质** 的 Hugging Face Docker Space 模板：
->
-> - `sub2api`、`PostgreSQL`、`Redis` 全部跑在 **同一个容器** 里。
-> - PostgreSQL / Redis 只在容器内部使用，不对外暴露。
-> - 数据库使用本地临时磁盘运行，**通过定时 `pg_dump` 备份到 Hugging Face Dataset Repo**。
->
-> **注意**：免费 Space 会休眠，且默认磁盘是临时的；这个模板更适合测试 / 演示，不适合生产环境。
-
-## 关键修复（v2）
-
-本版本修复了 Debian / Bookworm 上常见的 `externally-managed-environment`（PEP 668）报错：
-
-- 不再直接向系统 Python 安装依赖
-- 改为创建独立虚拟环境 `/home/user/venv`
-- `backup_worker.sh` 与 `supervisord.conf` 统一使用虚拟环境里的 Python
-
-## 在 Space Settings → Variables / Secrets 中至少配置
-
-```bash
-POSTGRES_PASSWORD=强密码
-JWT_SECRET=超长随机字符串
-TOTP_ENCRYPTION_KEY=超长随机字符串
-ADMIN_EMAIL=admin@example.com
-ADMIN_PASSWORD=强密码
-HF_TOKEN=你的 Hugging Face Write Token
-DATASET_REPO_ID=你的用户名/sub2api-backups
-```
+# Sub2API: 备份到 Dataset & 从 Dataset 自动恢复
+
+这个包包含 4 个文件：
+
+- `backup_to_dataset.py`：把 PostgreSQL 导出为 SQL.gz，上传到 Hugging Face Dataset Repo，并只保留最近 N 个备份。
+- `restore_from_dataset.py`：当本地数据库是空库时，从 Dataset Repo 下载最近一份 SQL.gz 并恢复。
+- `backup_worker.sh`：定时执行备份。
+- `requirements.txt`：安装 `huggingface_hub`。
+
+## 推荐接入方式
+
+1. 把 `backup_to_dataset.py`、`restore_from_dataset.py`、`backup_worker.sh`、`requirements.txt` 复制到 Space 仓库。 
+2. 在 Dockerfile 中把它们复制到镜像里，并在 venv 中安装 `huggingface_hub`。
+3. 在 `start.sh` 里：
+   - 记录 `fresh_db=true/false`
+   - 仅在 fresh DB 时执行 `restore_from_dataset.py`
+4. 在 `supervisord.conf` 里增加 `backup-worker` 进程。
+
+## 需要的 HF Space 环境变量
+
+### Secrets
+- `HF_TOKEN`：有 dataset repo 写权限的 Hugging Face token
+- `DATASET_REPO_ID`：例如 `gallyg/sub`
+
+### Variables
+- `BACKUP_INTERVAL_MINUTES=60`
+- `BACKUP_KEEP_LAST=10`
+- `AUTO_RESTORE_FROM_DATASET=true`
+
+## 说明
+
+- 适用于 Hugging Face 免费 Space 的“本地 PostgreSQL + 定时备份到 Dataset + 空库时自动恢复”模式。
+- 这不是块级持久化；如果 Space 在两次备份之间崩掉，仍可能丢失最近一小段新增数据。

backup_to_dataset.py

@@ -21,7 +21,6 @@ def env(name: str, default: str | None = None, required: bool = False) -> str:
 
 
 def run_pg_dump(tmp_sql: Path) -> None:
-    # 强制使用本地 PostgreSQL
     host = env("DATABASE_HOST", "127.0.0.1")
     port = env("DATABASE_PORT", "5432")
     user = env("DATABASE_USER", env("POSTGRES_USER", "sub2api"))
@@ -56,8 +55,7 @@ def upload_backup(gz_path: Path, metadata: dict) -> None:
     dataset_repo_id = env("DATASET_REPO_ID", required=True)
 
     api = HfApi(token=hf_token)
-    timestamp = metadata["timestamp_utc"]
-    remote_sql_path = f"postgres/{timestamp}.sql.gz"
+    remote_sql_path = metadata["remote_sql_path"]
     remote_latest_path = "postgres/latest.json"
 
     print(f"[backup] uploading {gz_path.name} -> {dataset_repo_id}:{remote_sql_path}")
@@ -66,7 +64,7 @@ def upload_backup(gz_path: Path, metadata: dict) -> None:
         path_in_repo=remote_sql_path,
         repo_id=dataset_repo_id,
         repo_type="dataset",
-        commit_message=f"backup: {timestamp}",
+        commit_message=f"backup: {metadata['timestamp_utc']}",
     )
 
     latest_tmp = gz_path.parent / "latest.json"
@@ -76,7 +74,7 @@ def upload_backup(gz_path: Path, metadata: dict) -> None:
         path_in_repo=remote_latest_path,
         repo_id=dataset_repo_id,
         repo_type="dataset",
-        commit_message=f"update latest backup metadata: {timestamp}",
+        commit_message=f"update latest backup metadata: {metadata['timestamp_utc']}",
     )
 
 
@@ -88,19 +86,10 @@ def prune_old_backups() -> None:
     fs = HfFileSystem(token=hf_token)
     api = HfApi(token=hf_token)
 
-    # 列出 dataset repo 下 postgres/ 的所有 sql.gz 备份
     pattern = f"datasets/{dataset_repo_id}/postgres/*.sql.gz"
     all_files = fs.glob(pattern)
-
-    # 转成 repo 内路径，如 postgres/20260320-120000.sql.gz
-    remote_paths = []
     prefix = f"datasets/{dataset_repo_id}/"
-    for p in all_files:
-        if p.startswith(prefix):
-            remote_paths.append(p[len(prefix):])
-
-    # 文件名天然按时间戳可排序
-    remote_paths = sorted(remote_paths)
+    remote_paths = sorted([p[len(prefix):] for p in all_files if p.startswith(prefix)])
 
     if len(remote_paths) <= keep_last:
         print(f"[backup] retention ok: {len(remote_paths)} <= {keep_last}")
@@ -108,7 +97,6 @@ def prune_old_backups() -> None:
 
     to_delete = remote_paths[:-keep_last]
     print(f"[backup] pruning {len(to_delete)} old backup(s)")
-
     operations = [CommitOperationDelete(path_in_repo=p) for p in to_delete]
     api.create_commit(
         repo_id=dataset_repo_id,
@@ -134,7 +122,6 @@ def main() -> int:
     try:
         run_pg_dump(sql_path)
         gzip_file(sql_path, gz_path)
-
         metadata = {
             "timestamp_utc": ts,
             "generated_at_iso": now.isoformat(),
@@ -143,12 +130,11 @@ def main() -> int:
             "database_name": env("DATABASE_DBNAME", env("POSTGRES_DB", "sub2api")),
             "dataset_repo_id": env("DATASET_REPO_ID", ""),
             "file_name": gz_path.name,
+            "remote_sql_path": f"postgres/{ts}.sql.gz",
             "file_size_bytes": gz_path.stat().st_size,
         }
-
         upload_backup(gz_path, metadata)
         prune_old_backups()
-
         print("[backup] done")
         return 0
     except Exception as exc:
@@ -163,4 +149,4 @@ def main() -> int:
 
 
 if __name__ == "__main__":
-    raise SystemExit(main())
+    raise SystemExit(main())

backup_worker.sh

@@ -10,4 +10,4 @@ while true; do
   "${VENV_PATH}/bin/python" "${APP_HOME}/backup_to_dataset.py" --once || true
   echo "[backup-worker] Sleeping ${INTERVAL_MINUTES} minute(s)..."
   sleep "$((INTERVAL_MINUTES * 60))"
-done
+done

requirements.txt

@@ -1 +1 @@
-huggingface_hub>=1.7.0
+huggingface_hub>=1.7.0

restore_from_dataset.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+import argparse
+import gzip
+import json
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+from huggingface_hub import HfFileSystem
+
+
+def env(name: str, default: str | None = None, required: bool = False) -> str:
+    value = os.getenv(name, default)
+    if required and not value:
+        raise RuntimeError(f"Missing required environment variable: {name}")
+    return value or ""
+
+
+def download_latest_metadata(fs: HfFileSystem, dataset_repo_id: str, workdir: Path) -> dict | None:
+    remote_latest = f"datasets/{dataset_repo_id}/postgres/latest.json"
+    local_latest = workdir / "latest.json"
+    try:
+        with fs.open(remote_latest, "rb") as src, local_latest.open("wb") as dst:
+            shutil.copyfileobj(src, dst)
+    except Exception as exc:
+        print(f"[restore] latest.json not found or unreadable: {exc}")
+        return None
+    return json.loads(local_latest.read_text(encoding="utf-8"))
+
+
+def download_backup(fs: HfFileSystem, dataset_repo_id: str, remote_sql_path: str, local_gz: Path) -> None:
+    remote_path = f"datasets/{dataset_repo_id}/{remote_sql_path}"
+    print(f"[restore] downloading {remote_path}")
+    with fs.open(remote_path, "rb") as src, local_gz.open("wb") as dst:
+        shutil.copyfileobj(src, dst)
+
+
+def gunzip_file(src: Path, dst: Path) -> None:
+    with gzip.open(src, "rb") as fin, dst.open("wb") as fout:
+        shutil.copyfileobj(fin, fout)
+
+
+def restore_sql(sql_path: Path) -> None:
+    host = env("DATABASE_HOST", "127.0.0.1")
+    port = env("DATABASE_PORT", "5432")
+    user = env("DATABASE_USER", env("POSTGRES_USER", "sub2api"))
+    password = env("DATABASE_PASSWORD", env("POSTGRES_PASSWORD", ""))
+    dbname = env("DATABASE_DBNAME", env("POSTGRES_DB", "sub2api"))
+
+    cmd = [
+        "psql",
+        "-h", host,
+        "-p", port,
+        "-U", user,
+        "-d", dbname,
+        "-v", "ON_ERROR_STOP=1",
+        "-f", str(sql_path),
+    ]
+    env_map = os.environ.copy()
+    env_map["PGPASSWORD"] = password
+    print(f"[restore] running: {' '.join(cmd[:-1])} <sql>")
+    subprocess.run(cmd, check=True, env=env_map)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--restore-latest", action="store_true")
+    parser.parse_args()
+
+    dataset_repo_id = env("DATASET_REPO_ID", "")
+    if not dataset_repo_id:
+        print("[restore] DATASET_REPO_ID not set, skipping restore")
+        return 0
+
+    hf_token = os.getenv("HF_TOKEN")
+    fs = HfFileSystem(token=hf_token) if hf_token else HfFileSystem()
+
+    workdir = Path("/tmp/sub2api_restore")
+    workdir.mkdir(parents=True, exist_ok=True)
+
+    try:
+        metadata = download_latest_metadata(fs, dataset_repo_id, workdir)
+        if not metadata:
+            print("[restore] no backup metadata available, skipping restore")
+            return 0
+
+        remote_sql_path = metadata.get("remote_sql_path") or f"postgres/{metadata['timestamp_utc']}.sql.gz"
+        gz_path = workdir / Path(remote_sql_path).name
+        sql_path = workdir / (gz_path.stem)
+
+        download_backup(fs, dataset_repo_id, remote_sql_path, gz_path)
+        gunzip_file(gz_path, sql_path)
+        restore_sql(sql_path)
+
+        marker = Path(env("SUB2API_DATA_DIR", "/app/data")) / "restore_last.json"
+        marker.parent.mkdir(parents=True, exist_ok=True)
+        marker.write_text(json.dumps(metadata, ensure_ascii=False, indent=2), encoding="utf-8")
+        print(f"[restore] restored backup {remote_sql_path}")
+        return 0
+    except Exception as exc:
+        print(f"[restore] failed: {exc}", file=sys.stderr)
+        return 1
+    finally:
+        for p in workdir.glob("*"):
+            try:
+                p.unlink()
+            except Exception:
+                pass
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

start_patch_and_instructions.txt

@@ -0,0 +1,52 @@
+# ===== 在 start.sh 中集成 restore / backup 的关键片段 =====
+
+# 1) 数据库环境变量里，继续保留本地 PostgreSQL
+export DATABASE_HOST="127.0.0.1"
+export DATABASE_PORT="5432"
+export DATABASE_USER="${POSTGRES_USER}"
+export DATABASE_PASSWORD="${POSTGRES_PASSWORD}"
+export DATABASE_DBNAME="${POSTGRES_DB}"
+export DATABASE_SSLMODE="disable"
+
+# 2) 初始化 PostgreSQL 前后，增加 fresh_db 标记
+fresh_db=false
+if [[ ! -s "$PGDATA/PG_VERSION" ]]; then
+  fresh_db=true
+fi
+
+# 3) 在创建数据库/角色成功后、停止 bootstrap PostgreSQL 之前，增加自动恢复
+if [[ "$fresh_db" == "true" && "${AUTO_RESTORE_FROM_DATASET:-true}" == "true" ]]; then
+  echo "[restore] fresh database detected, attempting restore from dataset..."
+  if [[ -x "$VENV_PATH/bin/python" ]]; then
+    "$VENV_PATH/bin/python" /app/restore_from_dataset.py --restore-latest || true
+  else
+    echo "[restore] python venv not found, skip restore"
+  fi
+fi
+
+# 4) supervisor 里增加 backup-worker
+[program:backup-worker]
+command=/bin/bash -lc "exec /usr/local/bin/backup_worker.sh"
+autostart=true
+autorestart=true
+startsecs=5
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+priority=40
+
+# 5) Dockerfile 里要复制这些文件并安装依赖
+COPY --chown=user:user backup_to_dataset.py /app/backup_to_dataset.py
+COPY --chown=user:user restore_from_dataset.py /app/restore_from_dataset.py
+COPY --chown=user:user backup_worker.sh /usr/local/bin/backup_worker.sh
+COPY --chown=user:user requirements.txt /app/requirements.txt
+
+# 6) HF Space Variables / Secrets 建议新增
+# Secrets:
+#   HF_TOKEN=你的 HF 写入 Token
+#   DATASET_REPO_ID=你的用户名/你的dataset仓库
+# Variables:
+#   BACKUP_INTERVAL_MINUTES=60
+#   BACKUP_KEEP_LAST=10
+#   AUTO_RESTORE_FROM_DATASET=true