Spaces:
Running on Zero
Running on Zero
feat(spaces): prune output dir before each generation
Browse filesWithout cleanup, every video lands in `~/comfyui/output/` and stays there
until the container restarts. The ephemeral disk is 150 GB and preload
already eats ~111 GB, so accumulating generations on the ~39 GB headroom
fills the disk and the replica goes unhealthy — observed today as a
stuck `RUNNING` Space with `replicas.current=0` and the iframe returning
`net::ERR_ABORTED`. Restart cleared it; this prevents the recurrence.
4 h TTL sweep at the top of `_on_generate`. Per-file `OSError` is
swallowed so one bad inode can't abort the cleanup. Tests cover
older-than-threshold deletion, recent-file retention, recursive descent
into ComfyUI's `LTX2.3/` subdir, missing-dir tolerance, and directory
preservation.
- app.py +32 -0
- tests/test_output_pruning.py +75 -0
app.py
CHANGED
|
@@ -871,6 +871,31 @@ def _seconds_to_frames(seconds: float, fps: int) -> int:
|
|
| 871 |
return max(9, int(round(float(seconds) * float(fps) / 8) * 8) + 1)
|
| 872 |
|
| 873 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 874 |
async def _on_generate(mode_name: str, *, progress: Any = None, **inputs: Any):
|
| 875 |
"""Generate handler — async generator yielding (status_html, video_path).
|
| 876 |
|
|
@@ -879,6 +904,13 @@ async def _on_generate(mode_name: str, *, progress: Any = None, **inputs: Any):
|
|
| 879 |
Spaces; we forward it to the backend so ComfyUI's per-step counter renders
|
| 880 |
a real progress bar instead of a generic Gradio spinner.
|
| 881 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 882 |
mode = modes.MODE_REGISTRY[mode_name]
|
| 883 |
|
| 884 |
fps = int(inputs.get("fps", 24))
|
|
|
|
| 871 |
return max(9, int(round(float(seconds) * float(fps) / 8) * 8) + 1)
|
| 872 |
|
| 873 |
|
| 874 |
+
def _prune_old_outputs(output_dir: pathlib.Path, max_age_seconds: int = 4 * 3600) -> int:
|
| 875 |
+
"""Delete files under *output_dir* older than *max_age_seconds*; return count.
|
| 876 |
+
|
| 877 |
+
HF Spaces ephemeral disk is 150 GB and preload already eats ~111 GB. Without
|
| 878 |
+
this sweep, generations accumulate in `~/comfyui/output/` until the disk
|
| 879 |
+
fills and the replica goes unhealthy (observed: stuck `RUNNING` with
|
| 880 |
+
`replicas.current=0`). Per-file OSError is swallowed so one bad file
|
| 881 |
+
doesn't abort a sweep that would otherwise free space.
|
| 882 |
+
"""
|
| 883 |
+
if not output_dir.exists():
|
| 884 |
+
return 0
|
| 885 |
+
cutoff = time.time() - max_age_seconds
|
| 886 |
+
deleted = 0
|
| 887 |
+
for f in output_dir.rglob("*"):
|
| 888 |
+
try:
|
| 889 |
+
if not f.is_file():
|
| 890 |
+
continue
|
| 891 |
+
if f.stat().st_mtime < cutoff:
|
| 892 |
+
f.unlink()
|
| 893 |
+
deleted += 1
|
| 894 |
+
except OSError:
|
| 895 |
+
continue
|
| 896 |
+
return deleted
|
| 897 |
+
|
| 898 |
+
|
| 899 |
async def _on_generate(mode_name: str, *, progress: Any = None, **inputs: Any):
|
| 900 |
"""Generate handler — async generator yielding (status_html, video_path).
|
| 901 |
|
|
|
|
| 904 |
Spaces; we forward it to the backend so ComfyUI's per-step counter renders
|
| 905 |
a real progress bar instead of a generic Gradio spinner.
|
| 906 |
"""
|
| 907 |
+
_comfy_dir_now = (
|
| 908 |
+
(pathlib.Path.home() / "comfyui")
|
| 909 |
+
if _on_spaces()
|
| 910 |
+
else pathlib.Path(__file__).parent / "comfyui"
|
| 911 |
+
)
|
| 912 |
+
_prune_old_outputs(_comfy_dir_now / "output")
|
| 913 |
+
|
| 914 |
mode = modes.MODE_REGISTRY[mode_name]
|
| 915 |
|
| 916 |
fps = int(inputs.get("fps", 24))
|
tests/test_output_pruning.py
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tests for the output-dir TTL sweeper.
|
| 2 |
+
|
| 3 |
+
ComfyUI writes every generated video/audio into `<comfy_dir>/output/...` and
|
| 4 |
+
nothing in this app cleans them up — left alone, they accumulate until the
|
| 5 |
+
Spaces ephemeral disk fills and the container goes unhealthy. `_prune_old_outputs`
|
| 6 |
+
sweeps files older than a TTL each time the user clicks Generate.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
import os
|
| 10 |
+
import pathlib
|
| 11 |
+
import time
|
| 12 |
+
|
| 13 |
+
import app
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def _touch(path: pathlib.Path, mtime: float | None = None) -> pathlib.Path:
|
| 17 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 18 |
+
path.write_bytes(b"x")
|
| 19 |
+
if mtime is not None:
|
| 20 |
+
os.utime(path, (mtime, mtime))
|
| 21 |
+
return path
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def test_prune_old_outputs_deletes_files_older_than_threshold(tmp_path: pathlib.Path) -> None:
|
| 25 |
+
stale = _touch(tmp_path / "old.mp4", mtime=time.time() - 5 * 3600)
|
| 26 |
+
|
| 27 |
+
app._prune_old_outputs(tmp_path, max_age_seconds=4 * 3600)
|
| 28 |
+
|
| 29 |
+
assert not stale.exists()
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def test_prune_old_outputs_keeps_files_younger_than_threshold(tmp_path: pathlib.Path) -> None:
|
| 33 |
+
fresh = _touch(tmp_path / "new.mp4", mtime=time.time() - 60)
|
| 34 |
+
|
| 35 |
+
app._prune_old_outputs(tmp_path, max_age_seconds=4 * 3600)
|
| 36 |
+
|
| 37 |
+
assert fresh.exists()
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def test_prune_old_outputs_returns_deleted_count(tmp_path: pathlib.Path) -> None:
|
| 41 |
+
_touch(tmp_path / "a.mp4", mtime=time.time() - 5 * 3600)
|
| 42 |
+
_touch(tmp_path / "b.mp4", mtime=time.time() - 6 * 3600)
|
| 43 |
+
_touch(tmp_path / "fresh.mp4", mtime=time.time() - 60)
|
| 44 |
+
|
| 45 |
+
deleted = app._prune_old_outputs(tmp_path, max_age_seconds=4 * 3600)
|
| 46 |
+
|
| 47 |
+
assert deleted == 2
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def test_prune_old_outputs_recurses_into_subdirs(tmp_path: pathlib.Path) -> None:
|
| 51 |
+
# ComfyUI writes under <output>/LTX2.3/<filename>.mp4 — the sweeper has
|
| 52 |
+
# to descend or it'd never touch real outputs.
|
| 53 |
+
stale = _touch(tmp_path / "LTX2.3" / "old.mp4", mtime=time.time() - 5 * 3600)
|
| 54 |
+
|
| 55 |
+
app._prune_old_outputs(tmp_path, max_age_seconds=4 * 3600)
|
| 56 |
+
|
| 57 |
+
assert not stale.exists()
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def test_prune_old_outputs_handles_missing_directory(tmp_path: pathlib.Path) -> None:
|
| 61 |
+
missing = tmp_path / "does-not-exist"
|
| 62 |
+
|
| 63 |
+
# No-op, no crash. Returning 0 is the only sensible answer.
|
| 64 |
+
assert app._prune_old_outputs(missing, max_age_seconds=4 * 3600) == 0
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def test_prune_old_outputs_leaves_directories_alone(tmp_path: pathlib.Path) -> None:
|
| 68 |
+
# The sweeper deletes *files*. Subdirs should remain so the next
|
| 69 |
+
# generation can write into the same path layout.
|
| 70 |
+
subdir = tmp_path / "LTX2.3"
|
| 71 |
+
subdir.mkdir()
|
| 72 |
+
|
| 73 |
+
app._prune_old_outputs(tmp_path, max_age_seconds=4 * 3600)
|
| 74 |
+
|
| 75 |
+
assert subdir.exists() and subdir.is_dir()
|