feat: package model weights, SAE checkpoints, and dynamic trajectories using Git LFS

.gitattributes

@@ -0,0 +1 @@
+*.pt filter=lfs diff=lfs merge=lfs -text

.github/workflows/hf_sync.yml

@@ -0,0 +1,54 @@
+name: Sync to Hugging Face Spaces
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch: # Allows manual syncing from the GitHub Action tab
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Debug Secret Presence
+        run: |
+          if [ -z "${{ secrets.HF_TOKEN }}" ]; then
+            echo "❌ HF_TOKEN is empty or missing in GitHub Repository Secrets!"
+            exit 1
+          else
+            echo "✅ HF_TOKEN is successfully configured in GitHub Repository Secrets (len: ${#HF_TOKEN})."
+          fi
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+
+      - name: Push to Hugging Face
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: |
+          # 1. Initialize Git LFS inside the runner
+          git lfs install
+          
+          # 2. Configure a named remote to allow Git LFS to authenticate correctly
+          git remote add hf https://sadhumitha-s:$HF_TOKEN@huggingface.co/spaces/sadhumitha-s/DT-Explorer
+          
+          # 3. Force push using HEAD:main to the hf remote
+          git push --force hf HEAD:main 2> push_err.txt || {
+            echo "=== HUGGING FACE PUSH ERROR LOG ==="
+            if [ -n "$HF_TOKEN" ]; then
+              sed "s/$HF_TOKEN/*****_TOKEN/g" push_err.txt
+            else
+              cat push_err.txt
+            fi
+            exit 1
+          }
+
+
+
+
+
+
+

.gitignore

@@ -26,12 +26,14 @@ venv/
 ENV/
 
 # Data and Models
-data/
-models/*.pt
-models/*.pth
+data/*
+!data/trajectories_demo.pt
+models/*
+!models/mini_dt.pt
+*.pth
 *.zip
 *.h5
-*.pt
+
 
 # Experiment Tracking
 wandb/
@@ -55,6 +57,8 @@ static/
 .venv
 
 /PRD.md
-artifacts/
+artifacts/*
+!artifacts/saes/
+!artifacts/saes/*
 scratch/
 *.log

Dockerfile

@@ -0,0 +1,44 @@
+FROM python:3.10-slim
+
+WORKDIR /app
+
+# Install system dependencies required for compilation and Gymnasium rendering
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    curl \
+    git \
+    libgl1-mesa-glx \
+    libglib2.0-0 \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy and install python dependencies first to cache this layer
+COPY requirements.txt .
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt
+
+# Create necessary directories
+RUN mkdir -p models data artifacts/saes
+
+# Copy trained model weights and SAE features
+COPY models/mini_dt.pt ./models/mini_dt.pt
+COPY artifacts/saes/ ./artifacts/saes/
+
+# Bake in the lightweight demo trajectories as the default dataset
+COPY data/trajectories_demo.pt ./data/trajectories.pt
+
+# Copy codebase
+COPY src/ ./src/
+
+# Expose default Streamlit port
+EXPOSE 8501
+
+# Streamlit configurations for production/cloud environments
+ENV STREAMLIT_SERVER_PORT=8501
+ENV STREAMLIT_SERVER_ADDRESS=0.0.0.0
+ENV STREAMLIT_SERVER_HEADLESS=true
+ENV STREAMLIT_SERVER_ENABLE_CORS=false
+ENV STREAMLIT_SERVER_ENABLE_XSRF=true
+ENV STREAMLIT_BROWSER_GATHER_USAGE_STATS=false
+
+# Start the dashboard application
+CMD ["streamlit", "run", "src/dashboard/app.py"]

Makefile

@@ -1,4 +1,4 @@
-.PHONY: setup train dashboard test clean
+.PHONY: setup train dashboard test clean deploy
 
 # Setup environment
 setup:
@@ -17,8 +17,44 @@ dashboard:
 test:
 	PYTHONPATH=. pytest tests/
 
+# Package and deploy to Hugging Face Spaces
+deploy:
+	@echo "1. Slicing trajectories to data/trajectories_demo.pt (with zero-hardcoded guardrail)..."
+	@echo "=========================================================="
+	@python3 -c ' \
+		import torch, os; \
+		data = torch.load("data/trajectories.pt", map_location="cpu", weights_only=False); \
+		count = len(data); \
+		torch.save(data[:count], "data/trajectories_demo.pt"); \
+		size_mb = os.path.getsize("data/trajectories_demo.pt") / (1024*1024); \
+		if size_mb >= 9.5: \
+			avg_size = size_mb / count; \
+			count = int(9.0 / avg_size); \
+			while count > 0: \
+				demo_data = data[:count]; \
+				torch.save(demo_data, "data/trajectories_demo.pt"); \
+				size_mb = os.path.getsize("data/trajectories_demo.pt") / (1024*1024); \
+				if size_mb < 9.5: \
+					break; \
+				count -= 1; \
+		print(f"Successfully packaged {count}/{len(data)} trajectories (Size: {size_mb:.2f} MB). Safely under 10MB limit."); \
+	'
+	@echo "Done."
+	@echo ""
+	@echo "=========================================================="
+	@echo "2. Staging and committing deployment assets..."
+	@echo "=========================================================="
+	@git add data/trajectories_demo.pt models/mini_dt.pt artifacts/saes/ .gitignore Dockerfile docker-compose.yml Makefile scripts/deploy.sh src/dashboard/app.py .github/workflows/hf_sync.yml
+	@git commit -m "feat: redeploy fresh model weights and demo trajectories" || echo "No new changes to commit."
+	@echo ""
+	@echo "=========================================================="
+	@echo "3. Pushing changes to Hugging Face Spaces ('hf' remote)..."
+	@echo "=========================================================="
+	@git push hf main || echo "Failed to push to 'hf' remote automatically. Please verify your Space git remote is named 'hf', or manually push to your target remote (e.g. 'git push origin main')."
+
 # Remove artifacts and cached files
 clean:
 	rm -rf data/*.pt models/*.pt artifacts/saes/*.pt
 	find . -type d -name "__pycache__" -exec rm -rf {} +
 	find . -type d -name ".pytest_cache" -exec rm -rf {} +
+

artifacts/saes/blocks_0_hook_resid_post_sae.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea054a846170dc71b71c8a65dd9091dfb670bf177641a63633b39281c270e658
+size 1056559

artifacts/saes/blocks_1_hook_resid_post_sae.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5736510ba07d2dc7c8481a7842011da02987ae4e875c68f28d06b2b0bc0f3c6c
+size 1056559

data/trajectories_demo.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b249672da20d83956eb4344a55eff64f1444a761fcd745a0873824e6f13ece4
+size 8599401

docker-compose.yml

@@ -0,0 +1,20 @@
+version: '3.8'
+
+services:
+  dt-explorer:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: dt_explorer
+    ports:
+      - "8501:8501"
+    volumes:
+      # Mount models and data directories to reflect updates in real-time
+      - ./models:/app/models
+      - ./data:/app/data
+      - ./artifacts:/app/artifacts
+    environment:
+      - PYTHONUNBUFFERED=1
+      # Optional: set wandb or neuronpedia tokens here if needed
+      # - WANDB_API_KEY=${WANDB_API_KEY}
+    restart: unless-stopped

models/mini_dt.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59f7d0f9708e6c2a22a56f369df757df57d2a0d750535ed4435f52dea89f5fcd
+size 4605691

scripts/deploy.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+set -e
+
+# DT-Explorer Automated Deployment Script
+# This script handles the raw bash workflow for solo researchers to update their hosted web app.
+
+echo "=========================================================="
+# 1. Slice heavy local trajectories into a lightweight demo set (zero-hardcoded dynamic scaling)
+echo "1. Slicing local trajectories down to data/trajectories_demo.pt..."
+echo "=========================================================="
+python3 -c '
+import torch, os
+data = torch.load("data/trajectories.pt", map_location="cpu", weights_only=False)
+count = len(data)
+# Try full dataset first
+torch.save(data[:count], "data/trajectories_demo.pt")
+size_mb = os.path.getsize("data/trajectories_demo.pt") / (1024*1024)
+
+if size_mb >= 9.5:
+    # Calculate average size per trajectory and estimate safe capacity
+    avg_size = size_mb / count
+    count = int(9.0 / avg_size) # Aim for ~9.0 MB to be safe
+    
+    # Verify and make minor adjustments if needed
+    while count > 0:
+        demo_data = data[:count]
+        torch.save(demo_data, "data/trajectories_demo.pt")
+        size_mb = os.path.getsize("data/trajectories_demo.pt") / (1024*1024)
+        if size_mb < 9.5:
+            break
+        count -= 1
+
+print(f"Successfully packaged {count}/{len(data)} trajectories (Size: {size_mb:.2f} MB). Safely under 10MB limit.")
+'
+echo "Done."
+echo ""
+
+echo "=========================================================="
+# 2. Stage model weights, SAE checkpoints, and configuration files
+echo "2. Staging deployment files in Git..."
+echo "=========================================================="
+git add data/trajectories_demo.pt models/mini_dt.pt artifacts/saes/ .gitignore Dockerfile docker-compose.yml Makefile scripts/deploy.sh src/dashboard/app.py .github/workflows/hf_sync.yml
+echo "Staged."
+echo ""
+
+echo "=========================================================="
+# 3. Commit changes locally
+echo "3. Committing staged changes..."
+echo "=========================================================="
+git commit -m "feat: redeploy fresh model weights and demo trajectories" || echo "No new changes to commit."
+echo ""
+
+echo "=========================================================="
+# 4. Push to GitHub (to trigger auto-sync to Hugging Face Space)
+echo "4. Pushing to GitHub (origin main)..."
+echo "=========================================================="
+git push origin main
+echo ""
+echo "Deployment successful! Check your Hugging Face Space or GitHub repository actions for the build status."

src/dashboard/app.py

@@ -25,11 +25,37 @@ st.title("DT-Explorer: Mechanistic Interpretability for DT")
 
 # Sidebar for loading model and data
 st.sidebar.header("Data & Model")
-model_path = st.sidebar.text_input("Model Path", "models/mini_dt.pt")
-data_path = st.sidebar.text_input("Trajectory Path", "data/trajectories.pt")
+
+# List available models in a secure dropdown to prevent Path Traversal
+models_dir = Path("models")
+available_models = []
+if models_dir.exists():
+    available_models = [str(p) for p in models_dir.glob("*.pt")]
+if not available_models:
+    available_models = ["models/mini_dt.pt"]
+model_path = st.sidebar.selectbox("Select Model Path", sorted(available_models))
+
+# List available datasets in a secure dropdown to prevent Path Traversal
+data_dir = Path("data")
+available_data = []
+if data_dir.exists():
+    available_data = [str(p) for p in data_dir.glob("*.pt")]
+if not available_data:
+    available_data = ["data/trajectories.pt"]
+data_path = st.sidebar.selectbox("Select Trajectory Path", sorted(available_data))
+
+# Validation check to guarantee path safety (Defense-in-depth)
+def is_safe_path(base_dir, path):
+    base_abs = Path(base_dir).resolve()
+    path_abs = Path(path).resolve()
+    return path_abs.parts[:len(base_abs.parts)] == base_abs.parts
 
 @st.cache_data
 def get_data(path):
+    if not is_safe_path("data", path):
+        st.sidebar.error("Access Denied: Invalid trajectory path.")
+        st.stop()
+        
     if not os.path.exists(path):
         st.sidebar.warning(f"Data not found at {path}. Please run training script.")
         return None
@@ -38,13 +64,17 @@ def get_data(path):
 
 @st.cache_resource
 def get_model(path, state_dim):
+    if not is_safe_path("models", path):
+        st.sidebar.error("Access Denied: Invalid model path.")
+        st.stop()
+        
     if not os.path.exists(path):
         st.sidebar.warning(f"Model not found at {path}. Using random init for demo.")
         return HookedDT.from_config(state_dim=state_dim, action_dim=7)
     
     model = HookedDT.from_config(state_dim=state_dim, action_dim=7)
     try:
-        # Load state dict (usually safe for weights_only=True, but let's be explicit)
+        # Load state dict (safe for weights_only=True)
         model.load_state_dict(torch.load(path, map_location="cpu", weights_only=True))
         model.eval()
     except Exception as e: