techfreakworm commited on
Commit
de6853a
·
verified ·
1 Parent(s): 5a81fc9

feat(spaces): prune output dir before each generation

Browse files

Without cleanup, every video lands in `~/comfyui/output/` and stays there
until the container restarts. The ephemeral disk is 150 GB and preload
already eats ~111 GB, so accumulating generations on the ~39 GB headroom
fills the disk and the replica goes unhealthy — observed today as a
stuck `RUNNING` Space with `replicas.current=0` and the iframe returning
`net::ERR_ABORTED`. Restart cleared it; this prevents the recurrence.

4 h TTL sweep at the top of `_on_generate`. Per-file `OSError` is
swallowed so one bad inode can't abort the cleanup. Tests cover
older-than-threshold deletion, recent-file retention, recursive descent
into ComfyUI's `LTX2.3/` subdir, missing-dir tolerance, and directory
preservation.

Files changed (2) hide show
  1. app.py +32 -0
  2. tests/test_output_pruning.py +75 -0
app.py CHANGED
@@ -871,6 +871,31 @@ def _seconds_to_frames(seconds: float, fps: int) -> int:
871
  return max(9, int(round(float(seconds) * float(fps) / 8) * 8) + 1)
872
 
873
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
874
  async def _on_generate(mode_name: str, *, progress: Any = None, **inputs: Any):
875
  """Generate handler — async generator yielding (status_html, video_path).
876
 
@@ -879,6 +904,13 @@ async def _on_generate(mode_name: str, *, progress: Any = None, **inputs: Any):
879
  Spaces; we forward it to the backend so ComfyUI's per-step counter renders
880
  a real progress bar instead of a generic Gradio spinner.
881
  """
 
 
 
 
 
 
 
882
  mode = modes.MODE_REGISTRY[mode_name]
883
 
884
  fps = int(inputs.get("fps", 24))
 
871
  return max(9, int(round(float(seconds) * float(fps) / 8) * 8) + 1)
872
 
873
 
874
+ def _prune_old_outputs(output_dir: pathlib.Path, max_age_seconds: int = 4 * 3600) -> int:
875
+ """Delete files under *output_dir* older than *max_age_seconds*; return count.
876
+
877
+ HF Spaces ephemeral disk is 150 GB and preload already eats ~111 GB. Without
878
+ this sweep, generations accumulate in `~/comfyui/output/` until the disk
879
+ fills and the replica goes unhealthy (observed: stuck `RUNNING` with
880
+ `replicas.current=0`). Per-file OSError is swallowed so one bad file
881
+ doesn't abort a sweep that would otherwise free space.
882
+ """
883
+ if not output_dir.exists():
884
+ return 0
885
+ cutoff = time.time() - max_age_seconds
886
+ deleted = 0
887
+ for f in output_dir.rglob("*"):
888
+ try:
889
+ if not f.is_file():
890
+ continue
891
+ if f.stat().st_mtime < cutoff:
892
+ f.unlink()
893
+ deleted += 1
894
+ except OSError:
895
+ continue
896
+ return deleted
897
+
898
+
899
  async def _on_generate(mode_name: str, *, progress: Any = None, **inputs: Any):
900
  """Generate handler — async generator yielding (status_html, video_path).
901
 
 
904
  Spaces; we forward it to the backend so ComfyUI's per-step counter renders
905
  a real progress bar instead of a generic Gradio spinner.
906
  """
907
+ _comfy_dir_now = (
908
+ (pathlib.Path.home() / "comfyui")
909
+ if _on_spaces()
910
+ else pathlib.Path(__file__).parent / "comfyui"
911
+ )
912
+ _prune_old_outputs(_comfy_dir_now / "output")
913
+
914
  mode = modes.MODE_REGISTRY[mode_name]
915
 
916
  fps = int(inputs.get("fps", 24))
tests/test_output_pruning.py ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Tests for the output-dir TTL sweeper.
2
+
3
+ ComfyUI writes every generated video/audio into `<comfy_dir>/output/...` and
4
+ nothing in this app cleans them up — left alone, they accumulate until the
5
+ Spaces ephemeral disk fills and the container goes unhealthy. `_prune_old_outputs`
6
+ sweeps files older than a TTL each time the user clicks Generate.
7
+ """
8
+
9
+ import os
10
+ import pathlib
11
+ import time
12
+
13
+ import app
14
+
15
+
16
+ def _touch(path: pathlib.Path, mtime: float | None = None) -> pathlib.Path:
17
+ path.parent.mkdir(parents=True, exist_ok=True)
18
+ path.write_bytes(b"x")
19
+ if mtime is not None:
20
+ os.utime(path, (mtime, mtime))
21
+ return path
22
+
23
+
24
+ def test_prune_old_outputs_deletes_files_older_than_threshold(tmp_path: pathlib.Path) -> None:
25
+ stale = _touch(tmp_path / "old.mp4", mtime=time.time() - 5 * 3600)
26
+
27
+ app._prune_old_outputs(tmp_path, max_age_seconds=4 * 3600)
28
+
29
+ assert not stale.exists()
30
+
31
+
32
+ def test_prune_old_outputs_keeps_files_younger_than_threshold(tmp_path: pathlib.Path) -> None:
33
+ fresh = _touch(tmp_path / "new.mp4", mtime=time.time() - 60)
34
+
35
+ app._prune_old_outputs(tmp_path, max_age_seconds=4 * 3600)
36
+
37
+ assert fresh.exists()
38
+
39
+
40
+ def test_prune_old_outputs_returns_deleted_count(tmp_path: pathlib.Path) -> None:
41
+ _touch(tmp_path / "a.mp4", mtime=time.time() - 5 * 3600)
42
+ _touch(tmp_path / "b.mp4", mtime=time.time() - 6 * 3600)
43
+ _touch(tmp_path / "fresh.mp4", mtime=time.time() - 60)
44
+
45
+ deleted = app._prune_old_outputs(tmp_path, max_age_seconds=4 * 3600)
46
+
47
+ assert deleted == 2
48
+
49
+
50
+ def test_prune_old_outputs_recurses_into_subdirs(tmp_path: pathlib.Path) -> None:
51
+ # ComfyUI writes under <output>/LTX2.3/<filename>.mp4 — the sweeper has
52
+ # to descend or it'd never touch real outputs.
53
+ stale = _touch(tmp_path / "LTX2.3" / "old.mp4", mtime=time.time() - 5 * 3600)
54
+
55
+ app._prune_old_outputs(tmp_path, max_age_seconds=4 * 3600)
56
+
57
+ assert not stale.exists()
58
+
59
+
60
+ def test_prune_old_outputs_handles_missing_directory(tmp_path: pathlib.Path) -> None:
61
+ missing = tmp_path / "does-not-exist"
62
+
63
+ # No-op, no crash. Returning 0 is the only sensible answer.
64
+ assert app._prune_old_outputs(missing, max_age_seconds=4 * 3600) == 0
65
+
66
+
67
+ def test_prune_old_outputs_leaves_directories_alone(tmp_path: pathlib.Path) -> None:
68
+ # The sweeper deletes *files*. Subdirs should remain so the next
69
+ # generation can write into the same path layout.
70
+ subdir = tmp_path / "LTX2.3"
71
+ subdir.mkdir()
72
+
73
+ app._prune_old_outputs(tmp_path, max_age_seconds=4 * 3600)
74
+
75
+ assert subdir.exists() and subdir.is_dir()