Spaces:
Runtime error
Runtime error
File size: 3,503 Bytes
7939f87 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 | #!/usr/bin/env bash
# Sync local repository content to a Hugging Face bucket.
#
# Usage examples:
# HF_TOKEN=hf_xxx ./cloud-run/sync_hf_bucket.sh
# ./cloud-run/sync_hf_bucket.sh --source ./kimodo --dry-run
# ./cloud-run/sync_hf_bucket.sh --dest hf://buckets/rydlrKE/movimento-bucket --include-build
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
DEST="hf://buckets/rydlrKE/movimento-bucket/kimodo"
SOURCE="."
DRY_RUN=false
INCLUDE_BUILD=false
VERBOSE=false
DELETE_MISSING=false
FILTER_FILE="$SCRIPT_DIR/hf_sync_filters.txt"
while [[ $# -gt 0 ]]; do
case "$1" in
--dest)
DEST="$2"
shift 2
;;
--source)
SOURCE="$2"
shift 2
;;
--dry-run)
DRY_RUN=true
shift
;;
--include-build)
INCLUDE_BUILD=true
shift
;;
--verbose)
VERBOSE=true
shift
;;
--delete)
DELETE_MISSING=true
shift
;;
--filter-file)
FILTER_FILE="$2"
shift 2
;;
-h|--help)
cat <<'EOF'
Sync local files to a Hugging Face bucket.
Options:
--source <path> Local source directory (default: .)
--dest <hf://...> HF bucket destination (default: hf://buckets/rydlrKE/movimento-bucket)
--dry-run Print planned actions without uploading
--include-build Include build/ artifacts (excluded by default)
--verbose Enable verbose sync output
--delete Delete destination files missing from source
--filter-file <path> Use custom include/exclude filter file
-h, --help Show this help
EOF
exit 0
;;
*)
echo "Unknown argument: $1" >&2
exit 1
;;
esac
done
if [[ ! -d "$SOURCE" ]]; then
echo "Source directory not found: $SOURCE" >&2
exit 1
fi
if ! command -v hf >/dev/null 2>&1; then
echo "Installing Hugging Face CLI..."
curl -LsSf https://hf.co/cli/install.sh | bash
fi
echo "Using HF CLI: $(command -v hf)"
hf --version
TOKEN="${HF_TOKEN:-${HUGGING_FACE_HUB_TOKEN:-${HF_HUB_TOKEN:-${HUGGINGFACEHUB_API_TOKEN:-}}}}"
SYNC_ARGS=()
if [[ -n "$TOKEN" ]]; then
SYNC_ARGS+=(--token "$TOKEN")
else
if ! hf auth whoami >/dev/null 2>&1; then
echo "No valid HF authentication found." >&2
echo "Set HF_TOKEN (or compatible HF token env var), or run: hf auth login --force" >&2
exit 1
fi
fi
SYNC_ARGS+=(
--exclude ".git/**"
--exclude ".pytest_cache/**"
--exclude ".mypy_cache/**"
--exclude ".ruff_cache/**"
--exclude ".nox/**"
--exclude ".tox/**"
--exclude ".venv/**"
--exclude ".tools/**"
--exclude "__pycache__/**"
--exclude "*/__pycache__/**"
--exclude "**/__pycache__/**"
--exclude "*.pyc"
--exclude "**/*.pyc"
--exclude "kimodo.egg-info/**"
--exclude "dist/**"
--exclude "docs/_build/**"
)
if [[ -n "$FILTER_FILE" ]]; then
if [[ ! -f "$FILTER_FILE" ]]; then
echo "Filter file not found: $FILTER_FILE" >&2
exit 1
fi
SYNC_ARGS+=(--filter-from "$FILTER_FILE")
fi
if [[ "$INCLUDE_BUILD" == "false" ]]; then
SYNC_ARGS+=(--exclude "build/**")
fi
if [[ "$DELETE_MISSING" == "true" ]]; then
SYNC_ARGS+=(--delete)
fi
if [[ "$DRY_RUN" == "true" ]]; then
SYNC_ARGS+=(--dry-run)
fi
if [[ "$VERBOSE" == "true" ]]; then
SYNC_ARGS+=(--verbose)
fi
echo "Syncing source: $SOURCE"
echo "Syncing destination: $DEST"
if [[ -n "$FILTER_FILE" ]]; then
echo "Using filter file: $FILTER_FILE"
fi
hf sync "$SOURCE" "$DEST" "${SYNC_ARGS[@]}"
echo "Sync complete."
|