Add files using upload-large-folder tool

distillation_data/compute_vae_latent.py

@@ -0,0 +1,97 @@
+from causvid.models.wan.wan_wrapper import WanVAEWrapper
+from causvid.util import launch_distributed_job
+import torch.distributed as dist
+import imageio.v3 as iio
+from tqdm import tqdm
+import argparse
+import torch
+import json
+import math
+import os
+
+torch.set_grad_enabled(False)
+
+
+def video_to_numpy(video_path):
+    """
+    Reads a video file and returns a NumPy array containing all frames.
+
+    :param video_path: Path to the video file.
+    :return: NumPy array of shape (num_frames, height, width, channels)
+    """
+    return iio.imread(video_path, plugin="pyav")  # Reads the entire video as a NumPy array
+
+
+def encode(self, videos: torch.Tensor) -> torch.Tensor:
+    device, dtype = videos[0].device, videos[0].dtype
+    scale = [self.mean.to(device=device, dtype=dtype),
+             1.0 / self.std.to(device=device, dtype=dtype)]
+    output = [
+        self.model.encode(u.unsqueeze(0), scale).float().squeeze(0)
+        for u in videos
+    ]
+
+    output = torch.stack(output, dim=0)
+    return output
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input_video_folder", type=str,
+                        help="Path to the folder containing input videos.")
+    parser.add_argument("--output_latent_folder", type=str,
+                        help="Path to the folder where output latents will be saved.")
+    parser.add_argument("--info_path", type=str,
+                        help="Path to the info file containing video metadata.")
+
+    args = parser.parse_args()
+
+    # Step 1: Setup the environment
+    torch.backends.cuda.matmul.allow_tf32 = True
+    torch.backends.cudnn.allow_tf32 = True
+    torch.set_grad_enabled(False)
+
+    # Step 2: Create the generator
+    launch_distributed_job()
+    device = torch.cuda.current_device()
+
+    with open(args.info_path, "r") as f:
+        video_info = json.load(f)
+
+    model = WanVAEWrapper().to(device=device, dtype=torch.bfloat16)
+
+    video_paths = sorted(list(video_info.keys()))
+
+    os.makedirs(args.output_latent_folder, exist_ok=True)
+
+    for index in tqdm(range(int(math.ceil(len(video_paths) / dist.get_world_size()))), disable=dist.get_rank() != 0):
+        global_index = index * dist.get_world_size() + dist.get_rank()
+        if global_index >= len(video_paths):
+            break
+
+        video_path = video_paths[global_index]
+        prompt = video_info[video_path]
+
+        try:
+            array = video_to_numpy(os.path.join(
+                args.input_video_folder, video_path))
+        except:
+            print(f"Failed to read video: {video_path}")
+            continue
+
+        video_tensor = torch.tensor(array, dtype=torch.float32, device=device).unsqueeze(0).permute(
+            0, 4, 1, 2, 3
+        ) / 255.0
+        video_tensor = video_tensor * 2 - 1
+        video_tensor = video_tensor.to(torch.bfloat16)
+        encoded_latents = encode(model, video_tensor).transpose(2, 1)
+
+        torch.save(
+            {prompt: encoded_latents.cpu().detach()},
+            os.path.join(args.output_latent_folder, f"{global_index:08d}.pt")
+        )
+    dist.barrier()
+
+
+if __name__ == "__main__":
+    main()

distillation_data/download_mixkit.py

@@ -0,0 +1,58 @@
+import argparse
+import os
+import tarfile
+from huggingface_hub import snapshot_download
+from concurrent.futures import ThreadPoolExecutor
+
+
+def extract_tar(tar_path, dest_dir):
+    """
+    Extracts a .tar file to the specified destination directory.
+    """
+    with tarfile.open(tar_path, 'r') as tar:
+        tar.extractall(path=dest_dir)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Download and extract dataset.")
+    parser.add_argument("--local_dir", type=str, default="/mnt/localssd/",
+                        help="Local directory to save the dataset.")
+    parser.add_argument("--repo_id", type=str,
+                        default="Languagebind/Open-Sora-Plan-v1.1.0", help="Hugging Face repository ID.")
+    parser.add_argument("--folder_name", type=str, default="all_mixkit",
+                        help="Folder name of the huggingface repo.")
+
+    args = parser.parse_args()
+
+    allow_patterns = [f"{args.folder_name}/*.tar"]
+
+    snapshot_download(
+        repo_id=args.repo_id,
+        local_dir=args.local_dir,
+        revision="main",          # or the branch/tag/commit you want
+        allow_patterns=allow_patterns,
+        repo_type="dataset"
+    )
+
+    # 4. Collect all .tar files recursively from the downloaded folder
+    tar_files = []
+    for root, dirs, files in os.walk(args.local_dir):
+        for file in files:
+            if file.endswith(".tar"):
+                tar_files.append(os.path.join(root, file))
+
+    # 5. Destination folder for extracted files
+    output_dir = os.path.join(args.local_dir, "videos")
+    os.makedirs(output_dir, exist_ok=True)
+
+    # 6. Extract each tar file in parallel
+    with ThreadPoolExecutor() as executor:
+        for tar_path in tar_files:
+            executor.submit(extract_tar, tar_path, output_dir)
+
+    print("All .tar files have been downloaded and extracted to:", output_dir)
+
+
+if __name__ == "__main__":
+    main()

distillation_data/ode_gen.sh

@@ -0,0 +1,6 @@
+export LD_LIBRARY_PATH=/usr/local/lib/:/opt/nccl/build/lib:/usr/local/cuda/lib64:/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
+
+torchrun --nnodes 8 --nproc_per_node=8 --rdzv_id=5235 \
+    --rdzv_backend=c10d \
+    --rdzv_endpoint $MASTER_ADDR causvid/models/wan/generate_ode_pairs.py \
+    --output_folder  mixkit_ode  --caption_path sample_dataset/mixkit_prompts.txt

distillation_data/process_mixkit.py

@@ -0,0 +1,161 @@
+# the following code is taken from FastVideo https://github.com/hao-ai-lab/FastVideo/tree/main
+# Apache-2.0 License
+
+import argparse
+import logging
+import time
+from concurrent.futures import ProcessPoolExecutor, as_completed
+from pathlib import Path
+
+import numpy as np
+from moviepy.editor import VideoFileClip
+from skimage.transform import resize
+from tqdm import tqdm
+from concurrent.futures import ThreadPoolExecutor
+
+# Configure logging
+logging.basicConfig(level=logging.INFO,
+                    format='%(asctime)s - %(levelname)s - %(message)s',
+                    handlers=[logging.FileHandler('video_processing.log')])
+
+
+def is_16_9_ratio(width: int, height: int, tolerance: float = 0.1) -> bool:
+    target_ratio = 16 / 9
+    actual_ratio = width / height
+    return abs(actual_ratio - target_ratio) <= (target_ratio * tolerance)
+
+
+def resize_video(args_tuple):
+    """
+    Resize a single video file.
+    args_tuple: (input_file, output_dir, width, height, fps)
+    """
+    input_file, output_dir, width, height, fps = args_tuple
+    video = None
+    resized = None
+    output_file = output_dir / f"{input_file.name}"
+
+    if output_file.exists():
+        output_file.unlink()
+
+    video = VideoFileClip(str(input_file))
+
+    if not is_16_9_ratio(video.w, video.h):
+        return (input_file.name, "skipped", "Not 16:9")
+
+    def process_frame(frame):
+        frame_float = frame.astype(float) / 255.0
+        resized = resize(frame_float, (height, width, 3),
+                         mode='reflect', anti_aliasing=True, preserve_range=True)
+        return (resized * 255).astype(np.uint8)
+
+    resized = video.fl_image(process_frame)
+
+    start_time = 0
+    end_time = (81 / 16)
+
+    # Crop the clip temporally using subclip
+    resized = resized.subclip(start_time, end_time)
+
+    # resized = resized.set_fps(fps)
+
+    resized.write_videofile(str(output_file),
+                            codec='libx264',
+                            audio_codec='aac',
+                            temp_audiofile=f'temp-audio-{input_file.stem}.m4a',
+                            remove_temp=True,
+                            verbose=False,
+                            logger=None,
+                            fps=fps)
+
+    return (input_file.name, "success", None)
+
+
+def process_folder(args):
+    input_path = Path(args.input_dir)
+    output_path = Path(args.output_dir)
+    output_path.mkdir(parents=True, exist_ok=True)
+
+    video_extensions = {'.mp4', '.avi', '.mov', '.mkv', '.webm'}
+    video_files = [f for f in input_path.iterdir() if f.is_file()
+                   and f.suffix.lower() in video_extensions]
+
+    if not video_files:
+        print(f"No video files found in {args.input_dir}")
+        return
+
+    print(f"Found {len(video_files)} videos")
+    print(f"Target: {args.width}x{args.height} at {args.fps}fps")
+
+    # Prepare arguments for parallel processing
+    process_args = [(video_file, output_path, args.width,
+                     args.height, args.fps) for video_file in video_files]
+
+    successful = 0
+    skipped = 0
+    failed = []
+
+    resize_video(process_args[0])
+
+    with tqdm(total=len(video_files), desc="Converting videos", dynamic_ncols=True) as pbar:
+        with ThreadPoolExecutor() as executor:
+            # Submit all tasks
+            future_to_file = {executor.submit(
+                resize_video, arg): arg[0] for arg in process_args}
+
+            # Process completed tasks
+            for future in as_completed(future_to_file):
+                filename, status, message = future.result()
+                if status == "success":
+                    successful += 1
+                elif status == "skipped":
+                    skipped += 1
+                else:
+                    failed.append((filename, message))
+                pbar.update(1)
+
+    # Print final summary
+    print(
+        f"\nDone! Processed: {successful}, Skipped: {skipped}, Failed: {len(failed)}")
+    if failed:
+        print("Failed files:")
+        for fname, error in failed:
+            print(f"- {fname}: {error}")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description='Batch resize videos to specified resolution and FPS (16:9 only)')
+    parser.add_argument('--input_dir', required=True,
+                        help='Input directory containing video files')
+    parser.add_argument('--output_dir', required=True,
+                        help='Output directory for processed videos')
+    parser.add_argument('--width', type=int, default=1280,
+                        help='Target width in pixels (default: 848)')
+    parser.add_argument('--height', type=int, default=720,
+                        help='Target height in pixels (default: 480)')
+    parser.add_argument('--fps', type=int, default=30,
+                        help='Target frames per second (default: 30)')
+    parser.add_argument('--log-level',
+                        choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'],
+                        default='INFO',
+                        help='Set the logging level (default: INFO)')
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    logging.getLogger().setLevel(getattr(logging, args.log_level))
+
+    if not Path(args.input_dir).exists():
+        logging.error(f"Input directory not found: {args.input_dir}")
+        return
+
+    start_time = time.time()
+    process_folder(args)
+    duration = time.time() - start_time
+    logging.info(f"Batch processing completed in {duration:.2f} seconds")
+
+
+if __name__ == "__main__":
+    main()

kill_processes.sh

@@ -0,0 +1,8 @@
+PIDS=$(ps aux | grep python | grep -v grep | awk '{print $2}')
+
+for PID in $PIDS; do
+	    # echo "Killing Python process with PID: $PID"
+	    kill -9 $PID
+	done
+
+	echo "All Python processes have been terminated."

minimal_inference/bidirectional_inference.py

@@ -0,0 +1,50 @@
+from causvid.models.wan.bidirectional_inference import BidirectionalInferencePipeline
+from huggingface_hub import hf_hub_download
+from diffusers.utils import export_to_video
+from causvid.data import TextDataset
+from omegaconf import OmegaConf
+from tqdm import tqdm
+import argparse
+import torch
+import os
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--config_path", type=str)
+parser.add_argument("--checkpoint_folder", type=str)
+parser.add_argument("--output_folder", type=str)
+parser.add_argument("--prompt_file_path", type=str)
+
+args = parser.parse_args()
+
+torch.backends.cuda.matmul.allow_tf32 = True
+torch.backends.cudnn.allow_tf32 = True
+
+torch.set_grad_enabled(False)
+
+config = OmegaConf.load(args.config_path)
+
+pipe = BidirectionalInferencePipeline(config, device="cuda")
+
+state_dict = torch.load(os.path.join(args.checkpoint_folder, "model.pt"), map_location="cpu")[
+    'generator']
+
+pipe.generator.load_state_dict(state_dict)
+
+pipe = pipe.to(device="cuda", dtype=torch.bfloat16)
+
+dataset = TextDataset(args.prompt_file_path)
+
+os.makedirs(args.output_folder, exist_ok=True)
+
+for index in tqdm(range(len(dataset))):
+    prompt = dataset[index]
+    video = pipe.inference(
+        noise=torch.randn(
+            1, 21, 16, 60, 104, generator=torch.Generator(device="cuda").manual_seed(42),
+            dtype=torch.bfloat16, device="cuda"
+        ),
+        text_prompts=[prompt]
+    )[0].permute(0, 2, 3, 1).cpu().numpy()
+
+    export_to_video(
+        video, os.path.join(args.output_folder, f"output_{index:03d}.mp4"), fps=16)

minimal_inference/longvideo_autoregressive_inference.py

@@ -0,0 +1,88 @@
+from causvid.models.wan.causal_inference import InferencePipeline
+from diffusers.utils import export_to_video
+from causvid.data import TextDataset
+from omegaconf import OmegaConf
+from tqdm import tqdm
+import numpy as np
+import argparse
+import torch
+import os
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--config_path", type=str)
+parser.add_argument("--checkpoint_folder", type=str)
+parser.add_argument("--prompt_file_path", type=str)
+parser.add_argument("--output_folder", type=str)
+parser.add_argument("--num_rollout", type=int, default=3)
+parser.add_argument("--num_overlap_frames", type=int, default=3)
+
+args = parser.parse_args()
+
+torch.set_grad_enabled(False)
+
+config = OmegaConf.load(args.config_path)
+
+pipeline = InferencePipeline(config, device="cuda")
+pipeline.to(device="cuda", dtype=torch.bfloat16)
+assert args.num_overlap_frames % pipeline.num_frame_per_block == 0, "num_overlap_frames must be divisible by num_frame_per_block"
+
+state_dict = torch.load(os.path.join(args.checkpoint_folder, "model.pt"), map_location="cpu")[
+    'generator']
+
+pipeline.generator.load_state_dict(
+    state_dict, strict=True
+)
+
+dataset = TextDataset(args.prompt_file_path)
+
+num_rollout = args.num_rollout
+
+os.makedirs(args.output_folder, exist_ok=True)
+
+
+def encode(self, videos: torch.Tensor) -> torch.Tensor:
+    device, dtype = videos[0].device, videos[0].dtype
+    scale = [self.mean.to(device=device, dtype=dtype),
+             1.0 / self.std.to(device=device, dtype=dtype)]
+    output = [
+        self.model.encode(u.unsqueeze(0), scale).float().squeeze(0)
+        for u in videos
+    ]
+
+    output = torch.stack(output, dim=0)
+    return output
+
+
+for prompt_index in tqdm(range(len(dataset))):
+    prompts = [dataset[prompt_index]]
+    start_latents = None
+    all_video = []
+
+    for rollout_index in range(num_rollout):
+        sampled_noise = torch.randn(
+            [1, 21, 16, 60, 104], device="cuda", dtype=torch.bfloat16
+        )
+
+        video, latents = pipeline.inference(
+            noise=sampled_noise,
+            text_prompts=prompts,
+            return_latents=True,
+            start_latents=start_latents
+        )
+
+        current_video = video[0].permute(0, 2, 3, 1).cpu().numpy()
+
+        start_frame = encode(pipeline.vae, (
+            video[:, -4 * (args.num_overlap_frames - 1) - 1:-4 * (args.num_overlap_frames - 1), :] * 2.0 - 1.0
+        ).transpose(2, 1).to(torch.bfloat16)).transpose(2, 1).to(torch.bfloat16)
+
+        start_latents = torch.cat(
+            [start_frame, latents[:, -(args.num_overlap_frames - 1):]], dim=1
+        )
+
+        all_video.append(current_video[:-(4 * (args.num_overlap_frames - 1) + 1)])
+
+    video = np.concatenate(all_video, axis=0)
+
+    export_to_video(
+        video, os.path.join(args.output_folder, f"long_video_output_{prompt_index:03d}.mp4"), fps=16)

output/wandb/debug-internal.log

@@ -0,0 +1,6 @@
+{"time":"2025-09-09T03:14:06.827346748Z","level":"INFO","msg":"stream: starting","core version":"0.21.3"}
+{"time":"2025-09-09T03:14:07.041670446Z","level":"INFO","msg":"stream: created new stream","id":"fvhxlznm"}
+{"time":"2025-09-09T03:14:07.04170638Z","level":"INFO","msg":"stream: started","id":"fvhxlznm"}
+{"time":"2025-09-09T03:14:07.04171651Z","level":"INFO","msg":"writer: started","stream_id":"fvhxlznm"}
+{"time":"2025-09-09T03:14:07.041726675Z","level":"INFO","msg":"handler: started","stream_id":"fvhxlznm"}
+{"time":"2025-09-09T03:14:07.041739753Z","level":"INFO","msg":"sender: started","stream_id":"fvhxlznm"}

output/wandb/debug.log

@@ -0,0 +1,22 @@
+2025-09-09 03:14:06,609 INFO    MainThread:1234529 [wandb_setup.py:_flush():81] Current SDK version is 0.21.3
+2025-09-09 03:14:06,609 INFO    MainThread:1234529 [wandb_setup.py:_flush():81] Configure stats pid to 1234529
+2025-09-09 03:14:06,609 INFO    MainThread:1234529 [wandb_setup.py:_flush():81] Loading settings from /home/yitongli/.config/wandb/settings
+2025-09-09 03:14:06,609 INFO    MainThread:1234529 [wandb_setup.py:_flush():81] Loading settings from /home/yitongli/CausVid/wandb/settings
+2025-09-09 03:14:06,609 INFO    MainThread:1234529 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2025-09-09 03:14:06,609 INFO    MainThread:1234529 [wandb_init.py:setup_run_log_directory():686] Logging user logs to ./output/wandb/run-20250909_031406-fvhxlznm/logs/debug.log
+2025-09-09 03:14:06,609 INFO    MainThread:1234529 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to ./output/wandb/run-20250909_031406-fvhxlznm/logs/debug-internal.log
+2025-09-09 03:14:06,609 INFO    MainThread:1234529 [wandb_init.py:init():813] calling init triggers
+2025-09-09 03:14:06,609 INFO    MainThread:1234529 [wandb_init.py:init():818] wandb.init called with sweep_config: {}
+config: {'model_name': 'causal_wan', 'generator_ckpt': 'pretrained_ode.pt', 'generator_grad': {'model': True}, 'denoising_step_list': [1000, 757, 522, 0], 'generator_task': 'causal_video', 'generator_fsdp_wrap_strategy': 'size', 'text_encoder_fsdp_wrap_strategy': 'size', 'mixed_precision': True, 'seed': 6553852, 'wandb_host': 'https://api.wandb.ai', 'wandb_key': '5409d3b960b01b25cec0f6abb5361b4022f0cc41', 'wandb_entity': 'liyitong-Tsinghua University', 'wandb_project': 'self-forcing', 'wandb_name': 'wan_causal_ode', 'sharding_strategy': 'hybrid_full', 'lr': 2e-06, 'beta1': 0.9, 'beta2': 0.999, 'data_path': '../mixkit_ode_lmdb', 'batch_size': 2, 'log_iters': 200, 'output_path': './output', 'distillation_loss': 'ode', 'gradient_checkpointing': True, 'num_frame_per_block': 3, 'warp_denoising_step': False, 'no_save': False, '_wandb': {}}
+2025-09-09 03:14:06,609 INFO    MainThread:1234529 [wandb_init.py:init():854] starting backend
+2025-09-09 03:14:06,816 INFO    MainThread:1234529 [wandb_init.py:init():857] sending inform_init request
+2025-09-09 03:14:06,821 INFO    MainThread:1234529 [wandb_init.py:init():865] backend started and connected
+2025-09-09 03:14:06,825 INFO    MainThread:1234529 [wandb_init.py:init():936] updated telemetry
+2025-09-09 03:14:06,830 INFO    MainThread:1234529 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout
+2025-09-09 03:14:07,304 INFO    MainThread:1234529 [wandb_init.py:init():1011] starting run threads in backend
+2025-09-09 03:14:07,415 INFO    MainThread:1234529 [wandb_run.py:_console_start():2494] atexit reg
+2025-09-09 03:14:07,415 INFO    MainThread:1234529 [wandb_run.py:_redirect():2342] redirect: wrap_raw
+2025-09-09 03:14:07,415 INFO    MainThread:1234529 [wandb_run.py:_redirect():2411] Wrapping output streams.
+2025-09-09 03:14:07,415 INFO    MainThread:1234529 [wandb_run.py:_redirect():2434] Redirects installed.
+2025-09-09 03:14:07,417 INFO    MainThread:1234529 [wandb_init.py:init():1057] run started, returning control to user process
+2025-09-09 03:14:07,902 INFO    MainThread:1234529 [wandb_run.py:_config_callback():1380] config_cb None None {'_wandb': {'code_path': 'source-self-forcing-causvid_train_ode.py'}}

output/wandb/run-20250908_062833-1l2wnyo3/files/config.yaml

@@ -0,0 +1,159 @@
+_wandb:
+    value:
+        cli_version: 0.21.3
+        code_path: source-self-forcing-causvid_train_ode.py
+        e:
+            aqq0exsxdyb4vz63hwlotexs67e6hip5:
+                args:
+                    - --config_path
+                    - configs/wan_causal_ode.yaml
+                codePath: causvid/train_ode.py
+                codePathLocal: causvid/train_ode.py
+                cpu_count: 48
+                cpu_count_logical: 96
+                cudaVersion: "12.8"
+                disk:
+                    /:
+                        total: "2079114358784"
+                        used: "1366588243968"
+                email: liyitong.thu@gmail.com
+                executable: /home/yitongli/miniconda3/envs/causvid/bin/python3.10
+                git:
+                    commit: adb6a5ecd07666b4d0290042915c8406e6d5ce22
+                    remote: https://github.com/tianweiy/CausVid.git
+                gpu: NVIDIA A100-SXM4-80GB
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-88ce35f2-86d5-a394-f90d-a52e2cd45949
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-1039594d-55ae-3eb3-339e-6d4d62c81d42
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-1f47056a-9c2c-8af1-8232-636203622830
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-38de1dff-a680-ec66-625b-49f2a31e3b42
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-712ffb29-5da2-92e5-99b7-9f148e8c1204
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-194ed489-5bfa-defc-85b1-812953e7d84a
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-21b1f0be-4c00-4af7-0649-9942ae8fd0e4
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-195cca5b-c8ee-790b-4d30-ae4d7aa93057
+                host: ip-172-31-3-169
+                memory:
+                    total: "1204521443328"
+                os: Linux-6.8.0-1028-aws-x86_64-with-glibc2.39
+                program: /home/yitongli/CausVid/causvid/train_ode.py
+                python: CPython 3.10.18
+                root: ./output
+                startedAt: "2025-09-08T06:28:33.977989Z"
+                writerId: aqq0exsxdyb4vz63hwlotexs67e6hip5
+        m: []
+        python_version: 3.10.18
+        t:
+            "1":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 71
+                - 83
+                - 105
+            "2":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 71
+                - 83
+                - 105
+            "3":
+                - 16
+                - 17
+            "4": 3.10.18
+            "5": 0.21.3
+            "6": 4.56.1
+            "12": 0.21.3
+            "13": linux-x86_64
+batch_size:
+    value: 2
+beta1:
+    value: 0.9
+beta2:
+    value: 0.999
+data_path:
+    value: ../mixkit_ode_lmdb
+denoising_step_list:
+    value:
+        - 1000
+        - 757
+        - 522
+        - 0
+distillation_loss:
+    value: ode
+generator_fsdp_wrap_strategy:
+    value: size
+generator_grad:
+    value:
+        model: true
+generator_task:
+    value: causal_video
+gradient_checkpointing:
+    value: true
+log_iters:
+    value: 200
+lr:
+    value: 2e-06
+mixed_precision:
+    value: true
+model_name:
+    value: causal_wan
+no_save:
+    value: false
+num_frame_per_block:
+    value: 3
+output_path:
+    value: ./output
+seed:
+    value: 8706203
+sharding_strategy:
+    value: hybrid_full
+text_encoder_fsdp_wrap_strategy:
+    value: size
+wandb_entity:
+    value: liyitong-Tsinghua University
+wandb_host:
+    value: https://api.wandb.ai
+wandb_key:
+    value: 5409d3b960b01b25cec0f6abb5361b4022f0cc41
+wandb_name:
+    value: wan_causal_ode
+wandb_project:
+    value: self-forcing
+warp_denoising_step:
+    value: false

output/wandb/run-20250908_062833-1l2wnyo3/files/output.log

@@ -0,0 +1,29 @@
+run dir: ./output/wandb/run-20250908_062833-1l2wnyo3/files
+Traceback (most recent call last):
+  File "/home/yitongli/CausVid/causvid/train_ode.py", line 211, in <module>
+    main()
+  File "/home/yitongli/CausVid/causvid/train_ode.py", line 204, in main
+    trainer = Trainer(config)
+  File "/home/yitongli/CausVid/causvid/train_ode.py", line 51, in __init__
+    self.distillation_model = ODERegression(config, device=self.device)
+  File "/home/yitongli/CausVid/causvid/ode_regression.py", line 39, in __init__
+    self.generator.enable_gradient_checkpointing()
+  File "/home/yitongli/CausVid/causvid/models/wan/wan_wrapper.py", line 113, in enable_gradient_checkpointing
+    self.model.enable_gradient_checkpointing()
+  File "/home/yitongli/miniconda3/envs/causvid/lib/python3.10/site-packages/diffusers/models/modeling_utils.py", line 310, in enable_gradient_checkpointing
+    self._set_gradient_checkpointing(enable=True, gradient_checkpointing_func=gradient_checkpointing_func)
+TypeError: CausalWanModel._set_gradient_checkpointing() got an unexpected keyword argument 'enable'
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/home/yitongli/CausVid/causvid/train_ode.py", line 211, in <module>
+[rank0]:     main()
+[rank0]:   File "/home/yitongli/CausVid/causvid/train_ode.py", line 204, in main
+[rank0]:     trainer = Trainer(config)
+[rank0]:   File "/home/yitongli/CausVid/causvid/train_ode.py", line 51, in __init__
+[rank0]:     self.distillation_model = ODERegression(config, device=self.device)
+[rank0]:   File "/home/yitongli/CausVid/causvid/ode_regression.py", line 39, in __init__
+[rank0]:     self.generator.enable_gradient_checkpointing()
+[rank0]:   File "/home/yitongli/CausVid/causvid/models/wan/wan_wrapper.py", line 113, in enable_gradient_checkpointing
+[rank0]:     self.model.enable_gradient_checkpointing()
+[rank0]:   File "/home/yitongli/miniconda3/envs/causvid/lib/python3.10/site-packages/diffusers/models/modeling_utils.py", line 310, in enable_gradient_checkpointing
+[rank0]:     self._set_gradient_checkpointing(enable=True, gradient_checkpointing_func=gradient_checkpointing_func)
+[rank0]: TypeError: CausalWanModel._set_gradient_checkpointing() got an unexpected keyword argument 'enable'

output/wandb/run-20250908_062833-1l2wnyo3/files/requirements.txt

@@ -0,0 +1,145 @@
+nvidia-cuda-nvrtc-cu12==12.8.93
+filelock==3.19.1
+aiofiles==24.1.0
+PyYAML==6.0.2
+httpx==0.28.1
+multidict==6.6.4
+tifffile==2025.5.10
+tzdata==2025.2
+urllib3==2.5.0
+decord==0.6.0
+certifi==2025.8.3
+setuptools==78.1.1
+websocket-client==1.8.0
+gitdb==4.0.12
+yarl==1.20.1
+tqdm==4.67.1
+open_clip_torch==3.1.0
+pyparsing==3.2.3
+fastapi==0.116.1
+nvidia-curand-cu12==10.3.9.90
+mdurl==0.1.2
+torchvision==0.23.0
+h11==0.16.0
+pytz==2025.2
+six==1.17.0
+nvidia-cufile-cu12==1.13.1.3
+aiohappyeyeballs==2.6.1
+wandb==0.21.3
+nvidia-nvjitlink-cu12==12.8.93
+cycler==0.12.1
+anyio==4.10.0
+scikit-image==0.25.2
+nvidia-cuda-runtime-cu12==12.8.90
+sentencepiece==0.2.1
+rich==14.1.0
+pydantic==2.11.7
+GitPython==3.1.45
+python-dateutil==2.9.0.post0
+accelerate==1.10.1
+proglog==0.1.12
+sentry-sdk==2.37.0
+nvidia-cusparselt-cu12==0.7.1
+dashscope==1.24.3
+platformdirs==4.4.0
+safehttpx==0.1.6
+fsspec==2025.9.0
+lazy_loader==0.4
+typing_extensions==4.15.0
+semantic-version==2.10.0
+numpy==1.24.4
+diffusers==0.35.1
+nvidia-cufft-cu12==11.3.3.83
+ruff==0.12.12
+nvidia-cudnn-cu12==9.10.2.21
+annotated-types==0.7.0
+decorator==4.4.2
+antlr4-python3-runtime==4.9.3
+psutil==7.0.0
+Brotli==1.1.0
+tomlkit==0.13.3
+httpcore==1.0.9
+kiwisolver==1.4.9
+nvidia-cusparse-cu12==12.5.8.93
+groovy==0.1.2
+protobuf==6.32.0
+orjson==3.11.3
+scipy==1.15.3
+regex==2025.9.1
+MarkupSafe==3.0.2
+av==13.1.0
+timm==1.0.19
+uvicorn==0.35.0
+wheel==0.45.1
+Pygments==2.19.2
+websockets==15.0.1
+lmdb==1.7.3
+sympy==1.14.0
+einops==0.8.1
+idna==3.10
+triton==3.4.0
+torch==2.8.0
+moviepy==1.0.3
+nvidia-nvtx-cu12==12.8.90
+matplotlib==3.10.6
+pillow==11.3.0
+charset-normalizer==3.4.3
+attrs==25.3.0
+aiosignal==1.4.0
+markdown-it-py==4.0.0
+requests==2.32.5
+typer==0.17.4
+huggingface-hub==0.34.4
+nvidia-nccl-cu12==2.27.3
+propcache==0.3.2
+opencv-python==4.11.0.86
+ffmpy==0.6.1
+jmespath==1.0.1
+botocore==1.40.25
+pydantic_core==2.33.2
+fonttools==4.59.2
+omegaconf==2.3.0
+pycparser==2.22
+mpmath==1.3.0
+flash_attn==2.8.3
+smmap==5.0.2
+gradio_client==1.12.1
+exceptiongroup==1.3.0
+cffi==1.17.1
+nvidia-cusolver-cu12==11.7.3.90
+contourpy==1.3.2
+cryptography==45.0.7
+nvidia-cuda-cupti-cu12==12.8.90
+wcwidth==0.2.13
+zipp==3.23.0
+safetensors==0.6.2
+gradio==5.44.1
+click==8.2.1
+frozenlist==1.7.0
+networkx==3.4.2
+s3transfer==0.13.1
+shellingham==1.5.4
+starlette==0.47.3
+packaging==25.0
+ftfy==6.3.1
+importlib_metadata==8.7.0
+transformers==4.56.1
+nvidia-cublas-cu12==12.8.4.1
+Jinja2==3.1.6
+pycocotools==2.0.10
+sniffio==1.3.1
+hf-xet==1.1.9
+typing-inspection==0.4.1
+pandas==2.3.2
+python-multipart==0.0.20
+aiohttp==3.12.15
+clip==1.0
+pydub==0.25.1
+easydict==1.13
+pip==25.2
+tokenizers==0.22.0
+imageio==2.37.0
+async-timeout==5.0.1
+boto3==1.40.25
+imageio-ffmpeg==0.6.0
+CausVid==0.0.1

output/wandb/run-20250908_062833-1l2wnyo3/files/wandb-metadata.json

@@ -0,0 +1,93 @@
+{
+  "os": "Linux-6.8.0-1028-aws-x86_64-with-glibc2.39",
+  "python": "CPython 3.10.18",
+  "startedAt": "2025-09-08T06:28:33.977989Z",
+  "args": [
+    "--config_path",
+    "configs/wan_causal_ode.yaml"
+  ],
+  "program": "/home/yitongli/CausVid/causvid/train_ode.py",
+  "codePath": "causvid/train_ode.py",
+  "codePathLocal": "causvid/train_ode.py",
+  "git": {
+    "remote": "https://github.com/tianweiy/CausVid.git",
+    "commit": "adb6a5ecd07666b4d0290042915c8406e6d5ce22"
+  },
+  "email": "liyitong.thu@gmail.com",
+  "root": "./output",
+  "host": "ip-172-31-3-169",
+  "executable": "/home/yitongli/miniconda3/envs/causvid/bin/python3.10",
+  "cpu_count": 48,
+  "cpu_count_logical": 96,
+  "gpu": "NVIDIA A100-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "2079114358784",
+      "used": "1366588243968"
+    }
+  },
+  "memory": {
+    "total": "1204521443328"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-88ce35f2-86d5-a394-f90d-a52e2cd45949"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-1039594d-55ae-3eb3-339e-6d4d62c81d42"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-1f47056a-9c2c-8af1-8232-636203622830"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-38de1dff-a680-ec66-625b-49f2a31e3b42"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-712ffb29-5da2-92e5-99b7-9f148e8c1204"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-194ed489-5bfa-defc-85b1-812953e7d84a"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-21b1f0be-4c00-4af7-0649-9942ae8fd0e4"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-195cca5b-c8ee-790b-4d30-ae4d7aa93057"
+    }
+  ],
+  "cudaVersion": "12.8",
+  "writerId": "aqq0exsxdyb4vz63hwlotexs67e6hip5"
+}

output/wandb/run-20250908_062833-1l2wnyo3/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":0},"_runtime":0}

output/wandb/run-20250908_062833-1l2wnyo3/logs/debug-core.log

@@ -0,0 +1,15 @@
+{"time":"2025-09-08T06:28:34.253598263Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpqgyjn6i1/port-3409973.txt","pid":3409973,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-09-08T06:28:34.257091825Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":3409973}
+{"time":"2025-09-08T06:28:34.256253546Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3409973-3410209-1263294975/socket","Net":"unix"}}
+{"time":"2025-09-08T06:28:34.391672548Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-09-08T06:28:34.406857393Z","level":"INFO","msg":"handleInformInit: received","streamId":"1l2wnyo3","id":"1(@)"}
+{"time":"2025-09-08T06:28:34.736516792Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"1l2wnyo3","id":"1(@)"}
+{"time":"2025-09-08T06:28:35.917810249Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-09-08T06:28:35.917878105Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-09-08T06:28:35.917865065Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-09-08T06:28:35.917988283Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-09-08T06:28:35.917976111Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3409973-3410209-1263294975/socket","Net":"unix"}}
+{"time":"2025-09-08T06:28:36.754084432Z","level":"ERROR","msg":"processOutgoingData: flush error","error":"write unix /tmp/wandb-3409973-3410209-1263294975/socket->@: use of closed network connection","id":"1(@)"}
+{"time":"2025-09-08T06:28:36.932139033Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-09-08T06:28:36.932153507Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-09-08T06:28:36.932164872Z","level":"INFO","msg":"server is closed"}

output/wandb/run-20250908_062833-1l2wnyo3/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2025-09-08T06:28:34.406945302Z","level":"INFO","msg":"stream: starting","core version":"0.21.3"}
+{"time":"2025-09-08T06:28:34.736482072Z","level":"INFO","msg":"stream: created new stream","id":"1l2wnyo3"}
+{"time":"2025-09-08T06:28:34.736512607Z","level":"INFO","msg":"stream: started","id":"1l2wnyo3"}
+{"time":"2025-09-08T06:28:34.736522488Z","level":"INFO","msg":"writer: started","stream_id":"1l2wnyo3"}
+{"time":"2025-09-08T06:28:34.7365292Z","level":"INFO","msg":"handler: started","stream_id":"1l2wnyo3"}
+{"time":"2025-09-08T06:28:34.736574661Z","level":"INFO","msg":"sender: started","stream_id":"1l2wnyo3"}
+{"time":"2025-09-08T06:28:35.91787343Z","level":"INFO","msg":"stream: closing","id":"1l2wnyo3"}
+{"time":"2025-09-08T06:28:36.804459652Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-09-08T06:28:36.9184071Z","level":"INFO","msg":"handler: closed","stream_id":"1l2wnyo3"}
+{"time":"2025-09-08T06:28:36.918464588Z","level":"INFO","msg":"sender: closed","stream_id":"1l2wnyo3"}
+{"time":"2025-09-08T06:28:36.918471349Z","level":"INFO","msg":"stream: closed","id":"1l2wnyo3"}

output/wandb/run-20250908_062833-1l2wnyo3/logs/debug.log

@@ -0,0 +1,24 @@
+2025-09-08 06:28:33,979 INFO    MainThread:3409973 [wandb_setup.py:_flush():81] Current SDK version is 0.21.3
+2025-09-08 06:28:33,979 INFO    MainThread:3409973 [wandb_setup.py:_flush():81] Configure stats pid to 3409973
+2025-09-08 06:28:33,979 INFO    MainThread:3409973 [wandb_setup.py:_flush():81] Loading settings from /home/yitongli/.config/wandb/settings
+2025-09-08 06:28:33,979 INFO    MainThread:3409973 [wandb_setup.py:_flush():81] Loading settings from /home/yitongli/CausVid/wandb/settings
+2025-09-08 06:28:33,979 INFO    MainThread:3409973 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2025-09-08 06:28:33,979 INFO    MainThread:3409973 [wandb_init.py:setup_run_log_directory():686] Logging user logs to ./output/wandb/run-20250908_062833-1l2wnyo3/logs/debug.log
+2025-09-08 06:28:33,979 INFO    MainThread:3409973 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to ./output/wandb/run-20250908_062833-1l2wnyo3/logs/debug-internal.log
+2025-09-08 06:28:33,979 INFO    MainThread:3409973 [wandb_init.py:init():813] calling init triggers
+2025-09-08 06:28:33,979 INFO    MainThread:3409973 [wandb_init.py:init():818] wandb.init called with sweep_config: {}
+config: {'model_name': 'causal_wan', 'generator_grad': {'model': True}, 'denoising_step_list': [1000, 757, 522, 0], 'generator_task': 'causal_video', 'generator_fsdp_wrap_strategy': 'size', 'text_encoder_fsdp_wrap_strategy': 'size', 'mixed_precision': True, 'seed': 8706203, 'wandb_host': 'https://api.wandb.ai', 'wandb_key': '5409d3b960b01b25cec0f6abb5361b4022f0cc41', 'wandb_entity': 'liyitong-Tsinghua University', 'wandb_project': 'self-forcing', 'wandb_name': 'wan_causal_ode', 'sharding_strategy': 'hybrid_full', 'lr': 2e-06, 'beta1': 0.9, 'beta2': 0.999, 'data_path': '../mixkit_ode_lmdb', 'batch_size': 2, 'log_iters': 200, 'output_path': './output', 'distillation_loss': 'ode', 'gradient_checkpointing': True, 'num_frame_per_block': 3, 'warp_denoising_step': False, 'no_save': False, '_wandb': {}}
+2025-09-08 06:28:33,979 INFO    MainThread:3409973 [wandb_init.py:init():854] starting backend
+2025-09-08 06:28:34,391 INFO    MainThread:3409973 [wandb_init.py:init():857] sending inform_init request
+2025-09-08 06:28:34,398 INFO    MainThread:3409973 [wandb_init.py:init():865] backend started and connected
+2025-09-08 06:28:34,400 INFO    MainThread:3409973 [wandb_init.py:init():936] updated telemetry
+2025-09-08 06:28:34,405 INFO    MainThread:3409973 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout
+2025-09-08 06:28:35,106 INFO    MainThread:3409973 [wandb_init.py:init():1011] starting run threads in backend
+2025-09-08 06:28:35,269 INFO    MainThread:3409973 [wandb_run.py:_console_start():2494] atexit reg
+2025-09-08 06:28:35,270 INFO    MainThread:3409973 [wandb_run.py:_redirect():2342] redirect: wrap_raw
+2025-09-08 06:28:35,270 INFO    MainThread:3409973 [wandb_run.py:_redirect():2411] Wrapping output streams.
+2025-09-08 06:28:35,270 INFO    MainThread:3409973 [wandb_run.py:_redirect():2434] Redirects installed.
+2025-09-08 06:28:35,273 INFO    MainThread:3409973 [wandb_init.py:init():1057] run started, returning control to user process
+2025-09-08 06:28:35,672 INFO    MainThread:3409973 [wandb_run.py:_config_callback():1380] config_cb None None {'_wandb': {'code_path': 'source-self-forcing-causvid_train_ode.py'}}
+2025-09-08 06:28:35,917 INFO    wandb-AsyncioManager-main:3409973 [service_client.py:_forward_responses():84] Reached EOF.
+2025-09-08 06:28:35,918 INFO    wandb-AsyncioManager-main:3409973 [mailbox.py:close():137] Closing mailbox, abandoning 2 handles.

output/wandb/run-20250908_064634-xd44venm/files/output.log

@@ -0,0 +1,26 @@
+run dir: ./output/wandb/run-20250908_064634-xd44venm/files
+Start training step 0...
+ cache a block wise causal mask with block size of 3 frames
+BlockMask(shape=(1, 1, 32768, 32768), sparsity=42.52%,
+(0, 0)
+████░░
+████░░
+████░░░░░░░░
+██████████░░
+██████████░░
+██████████░░░░░░░░
+████████████████░░
+████████████████░░
+████████████████░░░░░░░░
+██████████████████████░░
+██████████████████████░░
+██████████████████████░░░░░░░░
+████████████████████████████░░
+████████████████████████████░░
+████████████████████████████████░░
+████████████████████████████████░░
+████████████████████████████████░░░░░░░░
+████████████████████████████████████████
+████████████████████████████████████████
+████████████████████████████████████████
+)

output/wandb/run-20250908_064634-xd44venm/files/requirements.txt

@@ -0,0 +1,145 @@
+nvidia-cuda-nvrtc-cu12==12.8.93
+filelock==3.19.1
+aiofiles==24.1.0
+PyYAML==6.0.2
+httpx==0.28.1
+multidict==6.6.4
+tifffile==2025.5.10
+tzdata==2025.2
+urllib3==2.5.0
+decord==0.6.0
+certifi==2025.8.3
+setuptools==78.1.1
+websocket-client==1.8.0
+gitdb==4.0.12
+yarl==1.20.1
+tqdm==4.67.1
+open_clip_torch==3.1.0
+pyparsing==3.2.3
+fastapi==0.116.1
+nvidia-curand-cu12==10.3.9.90
+mdurl==0.1.2
+torchvision==0.23.0
+h11==0.16.0
+pytz==2025.2
+six==1.17.0
+nvidia-cufile-cu12==1.13.1.3
+aiohappyeyeballs==2.6.1
+wandb==0.21.3
+nvidia-nvjitlink-cu12==12.8.93
+cycler==0.12.1
+anyio==4.10.0
+scikit-image==0.25.2
+nvidia-cuda-runtime-cu12==12.8.90
+sentencepiece==0.2.1
+rich==14.1.0
+pydantic==2.11.7
+GitPython==3.1.45
+python-dateutil==2.9.0.post0
+accelerate==1.10.1
+proglog==0.1.12
+sentry-sdk==2.37.0
+nvidia-cusparselt-cu12==0.7.1
+dashscope==1.24.3
+platformdirs==4.4.0
+safehttpx==0.1.6
+fsspec==2025.9.0
+lazy_loader==0.4
+typing_extensions==4.15.0
+semantic-version==2.10.0
+numpy==1.24.4
+nvidia-cufft-cu12==11.3.3.83
+ruff==0.12.12
+nvidia-cudnn-cu12==9.10.2.21
+annotated-types==0.7.0
+decorator==4.4.2
+antlr4-python3-runtime==4.9.3
+psutil==7.0.0
+Brotli==1.1.0
+tomlkit==0.13.3
+httpcore==1.0.9
+kiwisolver==1.4.9
+nvidia-cusparse-cu12==12.5.8.93
+groovy==0.1.2
+protobuf==6.32.0
+orjson==3.11.3
+scipy==1.15.3
+regex==2025.9.1
+MarkupSafe==3.0.2
+av==13.1.0
+timm==1.0.19
+uvicorn==0.35.0
+wheel==0.45.1
+Pygments==2.19.2
+websockets==15.0.1
+lmdb==1.7.3
+sympy==1.14.0
+einops==0.8.1
+idna==3.10
+triton==3.4.0
+torch==2.8.0
+moviepy==1.0.3
+nvidia-nvtx-cu12==12.8.90
+matplotlib==3.10.6
+pillow==11.3.0
+charset-normalizer==3.4.3
+attrs==25.3.0
+aiosignal==1.4.0
+markdown-it-py==4.0.0
+requests==2.32.5
+typer==0.17.4
+huggingface-hub==0.34.4
+nvidia-nccl-cu12==2.27.3
+propcache==0.3.2
+opencv-python==4.11.0.86
+ffmpy==0.6.1
+jmespath==1.0.1
+botocore==1.40.25
+pydantic_core==2.33.2
+fonttools==4.59.2
+omegaconf==2.3.0
+pycparser==2.22
+mpmath==1.3.0
+flash_attn==2.8.3
+smmap==5.0.2
+gradio_client==1.12.1
+exceptiongroup==1.3.0
+cffi==1.17.1
+diffusers==0.31.0
+nvidia-cusolver-cu12==11.7.3.90
+contourpy==1.3.2
+cryptography==45.0.7
+nvidia-cuda-cupti-cu12==12.8.90
+wcwidth==0.2.13
+zipp==3.23.0
+safetensors==0.6.2
+gradio==5.44.1
+click==8.2.1
+frozenlist==1.7.0
+networkx==3.4.2
+s3transfer==0.13.1
+shellingham==1.5.4
+starlette==0.47.3
+packaging==25.0
+ftfy==6.3.1
+importlib_metadata==8.7.0
+transformers==4.56.1
+nvidia-cublas-cu12==12.8.4.1
+Jinja2==3.1.6
+pycocotools==2.0.10
+sniffio==1.3.1
+hf-xet==1.1.9
+typing-inspection==0.4.1
+pandas==2.3.2
+python-multipart==0.0.20
+aiohttp==3.12.15
+clip==1.0
+pydub==0.25.1
+easydict==1.13
+pip==25.2
+tokenizers==0.22.0
+imageio==2.37.0
+async-timeout==5.0.1
+boto3==1.40.25
+imageio-ffmpeg==0.6.0
+CausVid==0.0.1

output/wandb/run-20250908_064634-xd44venm/logs/debug-internal.log

@@ -0,0 +1,6 @@
+{"time":"2025-09-08T06:46:35.017276807Z","level":"INFO","msg":"stream: starting","core version":"0.21.3"}
+{"time":"2025-09-08T06:46:35.219581537Z","level":"INFO","msg":"stream: created new stream","id":"xd44venm"}
+{"time":"2025-09-08T06:46:35.219621685Z","level":"INFO","msg":"stream: started","id":"xd44venm"}
+{"time":"2025-09-08T06:46:35.219634304Z","level":"INFO","msg":"sender: started","stream_id":"xd44venm"}
+{"time":"2025-09-08T06:46:35.219635253Z","level":"INFO","msg":"handler: started","stream_id":"xd44venm"}
+{"time":"2025-09-08T06:46:35.219656757Z","level":"INFO","msg":"writer: started","stream_id":"xd44venm"}

output/wandb/run-20250908_064634-xd44venm/logs/debug.log

@@ -0,0 +1,22 @@
+2025-09-08 06:46:34,798 INFO    MainThread:3454108 [wandb_setup.py:_flush():81] Current SDK version is 0.21.3
+2025-09-08 06:46:34,798 INFO    MainThread:3454108 [wandb_setup.py:_flush():81] Configure stats pid to 3454108
+2025-09-08 06:46:34,798 INFO    MainThread:3454108 [wandb_setup.py:_flush():81] Loading settings from /home/yitongli/.config/wandb/settings
+2025-09-08 06:46:34,798 INFO    MainThread:3454108 [wandb_setup.py:_flush():81] Loading settings from /home/yitongli/CausVid/wandb/settings
+2025-09-08 06:46:34,798 INFO    MainThread:3454108 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2025-09-08 06:46:34,798 INFO    MainThread:3454108 [wandb_init.py:setup_run_log_directory():686] Logging user logs to ./output/wandb/run-20250908_064634-xd44venm/logs/debug.log
+2025-09-08 06:46:34,798 INFO    MainThread:3454108 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to ./output/wandb/run-20250908_064634-xd44venm/logs/debug-internal.log
+2025-09-08 06:46:34,798 INFO    MainThread:3454108 [wandb_init.py:init():813] calling init triggers
+2025-09-08 06:46:34,798 INFO    MainThread:3454108 [wandb_init.py:init():818] wandb.init called with sweep_config: {}
+config: {'model_name': 'causal_wan', 'generator_grad': {'model': True}, 'denoising_step_list': [1000, 757, 522, 0], 'generator_task': 'causal_video', 'generator_fsdp_wrap_strategy': 'size', 'text_encoder_fsdp_wrap_strategy': 'size', 'mixed_precision': True, 'seed': 6790074, 'wandb_host': 'https://api.wandb.ai', 'wandb_key': '5409d3b960b01b25cec0f6abb5361b4022f0cc41', 'wandb_entity': 'liyitong-Tsinghua University', 'wandb_project': 'self-forcing', 'wandb_name': 'wan_causal_ode', 'sharding_strategy': 'hybrid_full', 'lr': 2e-06, 'beta1': 0.9, 'beta2': 0.999, 'data_path': '../mixkit_ode_lmdb', 'batch_size': 2, 'log_iters': 200, 'output_path': './output', 'distillation_loss': 'ode', 'gradient_checkpointing': True, 'num_frame_per_block': 3, 'warp_denoising_step': False, 'no_save': False, '_wandb': {}}
+2025-09-08 06:46:34,798 INFO    MainThread:3454108 [wandb_init.py:init():854] starting backend
+2025-09-08 06:46:35,006 INFO    MainThread:3454108 [wandb_init.py:init():857] sending inform_init request
+2025-09-08 06:46:35,010 INFO    MainThread:3454108 [wandb_init.py:init():865] backend started and connected
+2025-09-08 06:46:35,015 INFO    MainThread:3454108 [wandb_init.py:init():936] updated telemetry
+2025-09-08 06:46:35,020 INFO    MainThread:3454108 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout
+2025-09-08 06:46:35,404 INFO    MainThread:3454108 [wandb_init.py:init():1011] starting run threads in backend
+2025-09-08 06:46:35,515 INFO    MainThread:3454108 [wandb_run.py:_console_start():2494] atexit reg
+2025-09-08 06:46:35,516 INFO    MainThread:3454108 [wandb_run.py:_redirect():2342] redirect: wrap_raw
+2025-09-08 06:46:35,516 INFO    MainThread:3454108 [wandb_run.py:_redirect():2411] Wrapping output streams.
+2025-09-08 06:46:35,516 INFO    MainThread:3454108 [wandb_run.py:_redirect():2434] Redirects installed.
+2025-09-08 06:46:35,518 INFO    MainThread:3454108 [wandb_init.py:init():1057] run started, returning control to user process
+2025-09-08 06:46:35,894 INFO    MainThread:3454108 [wandb_run.py:_config_callback():1380] config_cb None None {'_wandb': {'code_path': 'source-self-forcing-causvid_train_ode.py'}}

output/wandb/run-20250908_082236-gjh65qzq/files/config.yaml

@@ -0,0 +1,159 @@
+_wandb:
+    value:
+        cli_version: 0.21.3
+        code_path: source-self-forcing-causvid_train_ode.py
+        e:
+            ffoakmd0zn8f6sjv7nu2r58nmtyazqxg:
+                args:
+                    - --config_path
+                    - configs/wan_causal_ode.yaml
+                codePath: causvid/train_ode.py
+                codePathLocal: causvid/train_ode.py
+                cpu_count: 48
+                cpu_count_logical: 96
+                cudaVersion: "12.8"
+                disk:
+                    /:
+                        total: "2079114358784"
+                        used: "1376086970368"
+                email: liyitong.thu@gmail.com
+                executable: /home/yitongli/miniconda3/envs/causvid/bin/python3.10
+                git:
+                    commit: adb6a5ecd07666b4d0290042915c8406e6d5ce22
+                    remote: https://github.com/tianweiy/CausVid.git
+                gpu: NVIDIA A100-SXM4-80GB
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-88ce35f2-86d5-a394-f90d-a52e2cd45949
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-1039594d-55ae-3eb3-339e-6d4d62c81d42
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-1f47056a-9c2c-8af1-8232-636203622830
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-38de1dff-a680-ec66-625b-49f2a31e3b42
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-712ffb29-5da2-92e5-99b7-9f148e8c1204
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-194ed489-5bfa-defc-85b1-812953e7d84a
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-21b1f0be-4c00-4af7-0649-9942ae8fd0e4
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-195cca5b-c8ee-790b-4d30-ae4d7aa93057
+                host: ip-172-31-3-169
+                memory:
+                    total: "1204521443328"
+                os: Linux-6.8.0-1028-aws-x86_64-with-glibc2.39
+                program: /home/yitongli/CausVid/causvid/train_ode.py
+                python: CPython 3.10.18
+                root: ./output
+                startedAt: "2025-09-08T08:22:36.803576Z"
+                writerId: ffoakmd0zn8f6sjv7nu2r58nmtyazqxg
+        m: []
+        python_version: 3.10.18
+        t:
+            "1":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 71
+                - 83
+                - 105
+            "2":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 71
+                - 83
+                - 105
+            "3":
+                - 16
+                - 17
+            "4": 3.10.18
+            "5": 0.21.3
+            "6": 4.56.1
+            "12": 0.21.3
+            "13": linux-x86_64
+batch_size:
+    value: 2
+beta1:
+    value: 0.9
+beta2:
+    value: 0.999
+data_path:
+    value: ../mixkit_ode_lmdb
+denoising_step_list:
+    value:
+        - 1000
+        - 757
+        - 522
+        - 0
+distillation_loss:
+    value: ode
+generator_fsdp_wrap_strategy:
+    value: size
+generator_grad:
+    value:
+        model: true
+generator_task:
+    value: causal_video
+gradient_checkpointing:
+    value: true
+log_iters:
+    value: 200
+lr:
+    value: 2e-06
+mixed_precision:
+    value: true
+model_name:
+    value: causal_wan
+no_save:
+    value: false
+num_frame_per_block:
+    value: 3
+output_path:
+    value: ./output
+seed:
+    value: 7735925
+sharding_strategy:
+    value: hybrid_full
+text_encoder_fsdp_wrap_strategy:
+    value: size
+wandb_entity:
+    value: liyitong-Tsinghua University
+wandb_host:
+    value: https://api.wandb.ai
+wandb_key:
+    value: 5409d3b960b01b25cec0f6abb5361b4022f0cc41
+wandb_name:
+    value: wan_causal_ode
+wandb_project:
+    value: self-forcing
+warp_denoising_step:
+    value: false

output/wandb/run-20250908_082236-gjh65qzq/files/output.log

@@ -0,0 +1,36 @@
+run dir: ./output/wandb/run-20250908_082236-gjh65qzq/files
+KV inference with 3 frames per block
+ODERegression initialized.
+###########  torch.Size([2, 21, 16, 60, 104])
+Traceback (most recent call last):
+  File "/home/yitongli/CausVid/causvid/train_ode.py", line 229, in <module>
+    main()
+  File "/home/yitongli/CausVid/causvid/train_ode.py", line 223, in main
+    trainer.train()
+  File "/home/yitongli/CausVid/causvid/train_ode.py", line 187, in train
+    self.generate_video()
+  File "/home/yitongli/CausVid/causvid/train_ode.py", line 181, in generate_video
+    self.distillation_model.generate_video(ode_latent, conditional_dict)
+  File "/home/yitongli/CausVid/causvid/ode_regression.py", line 199, in generate_video
+    video = self.inference_pipeline.inference(
+  File "/home/yitongli/CausVid/causvid/models/wan/causal_inference.py", line 186, in inference
+    noisy_input = self.scheduler.add_noise(
+  File "/home/yitongli/CausVid/causvid/models/wan/flow_match.py", line 72, in add_noise
+    sample = (1 - sigma) * original_samples + sigma * noise
+RuntimeError: The size of tensor a (2) must match the size of tensor b (6) at non-singleton dimension 0
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/home/yitongli/CausVid/causvid/train_ode.py", line 229, in <module>
+[rank0]:     main()
+[rank0]:   File "/home/yitongli/CausVid/causvid/train_ode.py", line 223, in main
+[rank0]:     trainer.train()
+[rank0]:   File "/home/yitongli/CausVid/causvid/train_ode.py", line 187, in train
+[rank0]:     self.generate_video()
+[rank0]:   File "/home/yitongli/CausVid/causvid/train_ode.py", line 181, in generate_video
+[rank0]:     self.distillation_model.generate_video(ode_latent, conditional_dict)
+[rank0]:   File "/home/yitongli/CausVid/causvid/ode_regression.py", line 199, in generate_video
+[rank0]:     video = self.inference_pipeline.inference(
+[rank0]:   File "/home/yitongli/CausVid/causvid/models/wan/causal_inference.py", line 186, in inference
+[rank0]:     noisy_input = self.scheduler.add_noise(
+[rank0]:   File "/home/yitongli/CausVid/causvid/models/wan/flow_match.py", line 72, in add_noise
+[rank0]:     sample = (1 - sigma) * original_samples + sigma * noise
+[rank0]: RuntimeError: The size of tensor a (2) must match the size of tensor b (6) at non-singleton dimension 0

output/wandb/run-20250908_082236-gjh65qzq/files/requirements.txt

@@ -0,0 +1,145 @@
+nvidia-cuda-nvrtc-cu12==12.8.93
+filelock==3.19.1
+aiofiles==24.1.0
+PyYAML==6.0.2
+httpx==0.28.1
+multidict==6.6.4
+tifffile==2025.5.10
+tzdata==2025.2
+urllib3==2.5.0
+decord==0.6.0
+certifi==2025.8.3
+setuptools==78.1.1
+websocket-client==1.8.0
+gitdb==4.0.12
+yarl==1.20.1
+tqdm==4.67.1
+open_clip_torch==3.1.0
+pyparsing==3.2.3
+fastapi==0.116.1
+nvidia-curand-cu12==10.3.9.90
+mdurl==0.1.2
+torchvision==0.23.0
+h11==0.16.0
+pytz==2025.2
+six==1.17.0
+nvidia-cufile-cu12==1.13.1.3
+aiohappyeyeballs==2.6.1
+wandb==0.21.3
+nvidia-nvjitlink-cu12==12.8.93
+cycler==0.12.1
+anyio==4.10.0
+scikit-image==0.25.2
+nvidia-cuda-runtime-cu12==12.8.90
+sentencepiece==0.2.1
+rich==14.1.0
+pydantic==2.11.7
+GitPython==3.1.45
+python-dateutil==2.9.0.post0
+accelerate==1.10.1
+proglog==0.1.12
+sentry-sdk==2.37.0
+nvidia-cusparselt-cu12==0.7.1
+dashscope==1.24.3
+platformdirs==4.4.0
+safehttpx==0.1.6
+fsspec==2025.9.0
+lazy_loader==0.4
+typing_extensions==4.15.0
+semantic-version==2.10.0
+numpy==1.24.4
+nvidia-cufft-cu12==11.3.3.83
+ruff==0.12.12
+nvidia-cudnn-cu12==9.10.2.21
+annotated-types==0.7.0
+decorator==4.4.2
+antlr4-python3-runtime==4.9.3
+psutil==7.0.0
+Brotli==1.1.0
+tomlkit==0.13.3
+httpcore==1.0.9
+kiwisolver==1.4.9
+nvidia-cusparse-cu12==12.5.8.93
+groovy==0.1.2
+protobuf==6.32.0
+orjson==3.11.3
+scipy==1.15.3
+regex==2025.9.1
+MarkupSafe==3.0.2
+av==13.1.0
+timm==1.0.19
+uvicorn==0.35.0
+wheel==0.45.1
+Pygments==2.19.2
+websockets==15.0.1
+lmdb==1.7.3
+sympy==1.14.0
+einops==0.8.1
+idna==3.10
+triton==3.4.0
+torch==2.8.0
+moviepy==1.0.3
+nvidia-nvtx-cu12==12.8.90
+matplotlib==3.10.6
+pillow==11.3.0
+charset-normalizer==3.4.3
+attrs==25.3.0
+aiosignal==1.4.0
+markdown-it-py==4.0.0
+requests==2.32.5
+typer==0.17.4
+huggingface-hub==0.34.4
+nvidia-nccl-cu12==2.27.3
+propcache==0.3.2
+opencv-python==4.11.0.86
+ffmpy==0.6.1
+jmespath==1.0.1
+botocore==1.40.25
+pydantic_core==2.33.2
+fonttools==4.59.2
+omegaconf==2.3.0
+pycparser==2.22
+mpmath==1.3.0
+flash_attn==2.8.3
+smmap==5.0.2
+gradio_client==1.12.1
+exceptiongroup==1.3.0
+cffi==1.17.1
+diffusers==0.31.0
+nvidia-cusolver-cu12==11.7.3.90
+contourpy==1.3.2
+cryptography==45.0.7
+nvidia-cuda-cupti-cu12==12.8.90
+wcwidth==0.2.13
+zipp==3.23.0
+safetensors==0.6.2
+gradio==5.44.1
+click==8.2.1
+frozenlist==1.7.0
+networkx==3.4.2
+s3transfer==0.13.1
+shellingham==1.5.4
+starlette==0.47.3
+packaging==25.0
+ftfy==6.3.1
+importlib_metadata==8.7.0
+transformers==4.56.1
+nvidia-cublas-cu12==12.8.4.1
+Jinja2==3.1.6
+pycocotools==2.0.10
+sniffio==1.3.1
+hf-xet==1.1.9
+typing-inspection==0.4.1
+pandas==2.3.2
+python-multipart==0.0.20
+aiohttp==3.12.15
+clip==1.0
+pydub==0.25.1
+easydict==1.13
+pip==25.2
+tokenizers==0.22.0
+imageio==2.37.0
+async-timeout==5.0.1
+boto3==1.40.25
+imageio-ffmpeg==0.6.0
+CausVid==0.0.1

output/wandb/run-20250908_082236-gjh65qzq/files/wandb-metadata.json

@@ -0,0 +1,93 @@
+{
+  "os": "Linux-6.8.0-1028-aws-x86_64-with-glibc2.39",
+  "python": "CPython 3.10.18",
+  "startedAt": "2025-09-08T08:22:36.803576Z",
+  "args": [
+    "--config_path",
+    "configs/wan_causal_ode.yaml"
+  ],
+  "program": "/home/yitongli/CausVid/causvid/train_ode.py",
+  "codePath": "causvid/train_ode.py",
+  "codePathLocal": "causvid/train_ode.py",
+  "git": {
+    "remote": "https://github.com/tianweiy/CausVid.git",
+    "commit": "adb6a5ecd07666b4d0290042915c8406e6d5ce22"
+  },
+  "email": "liyitong.thu@gmail.com",
+  "root": "./output",
+  "host": "ip-172-31-3-169",
+  "executable": "/home/yitongli/miniconda3/envs/causvid/bin/python3.10",
+  "cpu_count": 48,
+  "cpu_count_logical": 96,
+  "gpu": "NVIDIA A100-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "2079114358784",
+      "used": "1376086970368"
+    }
+  },
+  "memory": {
+    "total": "1204521443328"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-88ce35f2-86d5-a394-f90d-a52e2cd45949"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-1039594d-55ae-3eb3-339e-6d4d62c81d42"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-1f47056a-9c2c-8af1-8232-636203622830"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-38de1dff-a680-ec66-625b-49f2a31e3b42"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-712ffb29-5da2-92e5-99b7-9f148e8c1204"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-194ed489-5bfa-defc-85b1-812953e7d84a"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-21b1f0be-4c00-4af7-0649-9942ae8fd0e4"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-195cca5b-c8ee-790b-4d30-ae4d7aa93057"
+    }
+  ],
+  "cudaVersion": "12.8",
+  "writerId": "ffoakmd0zn8f6sjv7nu2r58nmtyazqxg"
+}

output/wandb/run-20250908_082236-gjh65qzq/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":98},"_runtime":98}

output/wandb/run-20250908_082236-gjh65qzq/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2025-09-08T08:22:36.825042468Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpk_l8zwe3/port-3836172.txt","pid":3836172,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-09-08T08:22:36.826033664Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":3836172}
+{"time":"2025-09-08T08:22:36.825970015Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3836172-3836464-2285782236/socket","Net":"unix"}}
+{"time":"2025-09-08T08:22:37.011146283Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-09-08T08:22:37.01985984Z","level":"INFO","msg":"handleInformInit: received","streamId":"gjh65qzq","id":"1(@)"}
+{"time":"2025-09-08T08:22:37.227697259Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"gjh65qzq","id":"1(@)"}
+{"time":"2025-09-08T08:24:15.728157025Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-09-08T08:24:15.728207682Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-09-08T08:24:15.728220917Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-09-08T08:24:15.728290134Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-09-08T08:24:15.728328201Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3836172-3836464-2285782236/socket","Net":"unix"}}
+{"time":"2025-09-08T08:24:16.267531359Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-09-08T08:24:16.267567371Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-09-08T08:24:16.26757774Z","level":"INFO","msg":"server is closed"}

output/wandb/run-20250908_082236-gjh65qzq/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2025-09-08T08:22:37.019990235Z","level":"INFO","msg":"stream: starting","core version":"0.21.3"}
+{"time":"2025-09-08T08:22:37.227508836Z","level":"INFO","msg":"stream: created new stream","id":"gjh65qzq"}
+{"time":"2025-09-08T08:22:37.227691977Z","level":"INFO","msg":"stream: started","id":"gjh65qzq"}
+{"time":"2025-09-08T08:22:37.227709815Z","level":"INFO","msg":"writer: started","stream_id":"gjh65qzq"}
+{"time":"2025-09-08T08:22:37.227733552Z","level":"INFO","msg":"handler: started","stream_id":"gjh65qzq"}
+{"time":"2025-09-08T08:22:37.227762439Z","level":"INFO","msg":"sender: started","stream_id":"gjh65qzq"}
+{"time":"2025-09-08T08:24:15.728213817Z","level":"INFO","msg":"stream: closing","id":"gjh65qzq"}
+{"time":"2025-09-08T08:24:16.029737674Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-09-08T08:24:16.167542431Z","level":"INFO","msg":"handler: closed","stream_id":"gjh65qzq"}
+{"time":"2025-09-08T08:24:16.167636838Z","level":"INFO","msg":"sender: closed","stream_id":"gjh65qzq"}
+{"time":"2025-09-08T08:24:16.167654281Z","level":"INFO","msg":"stream: closed","id":"gjh65qzq"}

output/wandb/run-20250908_082236-gjh65qzq/logs/debug.log

@@ -0,0 +1,24 @@
+2025-09-08 08:22:36,804 INFO    MainThread:3836172 [wandb_setup.py:_flush():81] Current SDK version is 0.21.3
+2025-09-08 08:22:36,804 INFO    MainThread:3836172 [wandb_setup.py:_flush():81] Configure stats pid to 3836172
+2025-09-08 08:22:36,804 INFO    MainThread:3836172 [wandb_setup.py:_flush():81] Loading settings from /home/yitongli/.config/wandb/settings
+2025-09-08 08:22:36,805 INFO    MainThread:3836172 [wandb_setup.py:_flush():81] Loading settings from /home/yitongli/CausVid/wandb/settings
+2025-09-08 08:22:36,805 INFO    MainThread:3836172 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2025-09-08 08:22:36,805 INFO    MainThread:3836172 [wandb_init.py:setup_run_log_directory():686] Logging user logs to ./output/wandb/run-20250908_082236-gjh65qzq/logs/debug.log
+2025-09-08 08:22:36,805 INFO    MainThread:3836172 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to ./output/wandb/run-20250908_082236-gjh65qzq/logs/debug-internal.log
+2025-09-08 08:22:36,805 INFO    MainThread:3836172 [wandb_init.py:init():813] calling init triggers
+2025-09-08 08:22:36,805 INFO    MainThread:3836172 [wandb_init.py:init():818] wandb.init called with sweep_config: {}
+config: {'model_name': 'causal_wan', 'generator_grad': {'model': True}, 'denoising_step_list': [1000, 757, 522, 0], 'generator_task': 'causal_video', 'generator_fsdp_wrap_strategy': 'size', 'text_encoder_fsdp_wrap_strategy': 'size', 'mixed_precision': True, 'seed': 7735925, 'wandb_host': 'https://api.wandb.ai', 'wandb_key': '5409d3b960b01b25cec0f6abb5361b4022f0cc41', 'wandb_entity': 'liyitong-Tsinghua University', 'wandb_project': 'self-forcing', 'wandb_name': 'wan_causal_ode', 'sharding_strategy': 'hybrid_full', 'lr': 2e-06, 'beta1': 0.9, 'beta2': 0.999, 'data_path': '../mixkit_ode_lmdb', 'batch_size': 2, 'log_iters': 200, 'output_path': './output', 'distillation_loss': 'ode', 'gradient_checkpointing': True, 'num_frame_per_block': 3, 'warp_denoising_step': False, 'no_save': False, '_wandb': {}}
+2025-09-08 08:22:36,805 INFO    MainThread:3836172 [wandb_init.py:init():854] starting backend
+2025-09-08 08:22:37,011 INFO    MainThread:3836172 [wandb_init.py:init():857] sending inform_init request
+2025-09-08 08:22:37,015 INFO    MainThread:3836172 [wandb_init.py:init():865] backend started and connected
+2025-09-08 08:22:37,018 INFO    MainThread:3836172 [wandb_init.py:init():936] updated telemetry
+2025-09-08 08:22:37,022 INFO    MainThread:3836172 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout
+2025-09-08 08:22:37,413 INFO    MainThread:3836172 [wandb_init.py:init():1011] starting run threads in backend
+2025-09-08 08:22:37,526 INFO    MainThread:3836172 [wandb_run.py:_console_start():2494] atexit reg
+2025-09-08 08:22:37,526 INFO    MainThread:3836172 [wandb_run.py:_redirect():2342] redirect: wrap_raw
+2025-09-08 08:22:37,526 INFO    MainThread:3836172 [wandb_run.py:_redirect():2411] Wrapping output streams.
+2025-09-08 08:22:37,526 INFO    MainThread:3836172 [wandb_run.py:_redirect():2434] Redirects installed.
+2025-09-08 08:22:37,528 INFO    MainThread:3836172 [wandb_init.py:init():1057] run started, returning control to user process
+2025-09-08 08:22:37,914 INFO    MainThread:3836172 [wandb_run.py:_config_callback():1380] config_cb None None {'_wandb': {'code_path': 'source-self-forcing-causvid_train_ode.py'}}
+2025-09-08 08:24:15,728 INFO    wandb-AsyncioManager-main:3836172 [service_client.py:_forward_responses():84] Reached EOF.
+2025-09-08 08:24:15,729 INFO    wandb-AsyncioManager-main:3836172 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.

output/wandb/run-20250908_091215-tz5j30tc/files/config.yaml

@@ -0,0 +1,160 @@
+_wandb:
+    value:
+        cli_version: 0.21.3
+        code_path: source-self-forcing-causvid_train_ode.py
+        e:
+            dzykhs2dp41ohf8z5ba4wgauzbq2dbq2:
+                args:
+                    - --config_path
+                    - configs/wan_causal_ode.yaml
+                codePath: causvid/train_ode.py
+                codePathLocal: causvid/train_ode.py
+                cpu_count: 48
+                cpu_count_logical: 96
+                cudaVersion: "12.8"
+                disk:
+                    /:
+                        total: "2079114358784"
+                        used: "1400048283648"
+                email: liyitong.thu@gmail.com
+                executable: /home/yitongli/miniconda3/envs/causvid/bin/python3.10
+                git:
+                    commit: adb6a5ecd07666b4d0290042915c8406e6d5ce22
+                    remote: https://github.com/tianweiy/CausVid.git
+                gpu: NVIDIA A100-SXM4-80GB
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-88ce35f2-86d5-a394-f90d-a52e2cd45949
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-1039594d-55ae-3eb3-339e-6d4d62c81d42
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-1f47056a-9c2c-8af1-8232-636203622830
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-38de1dff-a680-ec66-625b-49f2a31e3b42
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-712ffb29-5da2-92e5-99b7-9f148e8c1204
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-194ed489-5bfa-defc-85b1-812953e7d84a
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-21b1f0be-4c00-4af7-0649-9942ae8fd0e4
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-195cca5b-c8ee-790b-4d30-ae4d7aa93057
+                host: ip-172-31-3-169
+                memory:
+                    total: "1204521443328"
+                os: Linux-6.8.0-1028-aws-x86_64-with-glibc2.39
+                program: /home/yitongli/CausVid/causvid/train_ode.py
+                python: CPython 3.10.18
+                root: ./output
+                startedAt: "2025-09-08T09:12:15.567002Z"
+                writerId: dzykhs2dp41ohf8z5ba4wgauzbq2dbq2
+        m: []
+        python_version: 3.10.18
+        t:
+            "1":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 71
+                - 83
+                - 105
+            "2":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 71
+                - 83
+                - 105
+            "3":
+                - 16
+                - 17
+                - 61
+            "4": 3.10.18
+            "5": 0.21.3
+            "6": 4.56.1
+            "12": 0.21.3
+            "13": linux-x86_64
+batch_size:
+    value: 2
+beta1:
+    value: 0.9
+beta2:
+    value: 0.999
+data_path:
+    value: ../mixkit_ode_lmdb
+denoising_step_list:
+    value:
+        - 1000
+        - 757
+        - 522
+        - 0
+distillation_loss:
+    value: ode
+generator_fsdp_wrap_strategy:
+    value: size
+generator_grad:
+    value:
+        model: true
+generator_task:
+    value: causal_video
+gradient_checkpointing:
+    value: true
+log_iters:
+    value: 200
+lr:
+    value: 2e-06
+mixed_precision:
+    value: true
+model_name:
+    value: causal_wan
+no_save:
+    value: false
+num_frame_per_block:
+    value: 3
+output_path:
+    value: ./output
+seed:
+    value: 550819
+sharding_strategy:
+    value: hybrid_full
+text_encoder_fsdp_wrap_strategy:
+    value: size
+wandb_entity:
+    value: liyitong-Tsinghua University
+wandb_host:
+    value: https://api.wandb.ai
+wandb_key:
+    value: 5409d3b960b01b25cec0f6abb5361b4022f0cc41
+wandb_name:
+    value: wan_causal_ode
+wandb_project:
+    value: self-forcing
+warp_denoising_step:
+    value: false

output/wandb/run-20250908_091215-tz5j30tc/files/output.log

@@ -0,0 +1,53 @@
+run dir: ./output/wandb/run-20250908_091215-tz5j30tc/files
+KV inference with 3 frames per block
+ODERegression initialized.
+ cache a block wise causal mask with block size of 3 frames
+BlockMask(shape=(1, 1, 32768, 32768), sparsity=42.52%,
+(0, 0)
+████░░
+████░░
+████░░░░░░░░
+██████████░░
+██████████░░
+██████████░░░░░░░░
+████████████████░░
+████████████████░░
+████████████████░░░░░░░░
+██████████████████████░░
+██████████████████████░░
+██████████████████████░░░░░░░░
+████████████████████████████░░
+████████████████████████████░░
+████████████████████████████████░░
+████████████████████████████████░░
+████████████████████████████████░░░░░░░░
+████████████████████████████████████████
+████████████████████████████████████████
+████████████████████████████████████████
+)
+arrive
+Start gathering distributed model states...
+/home/yitongli/miniconda3/envs/causvid/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:678: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Model saved to ./output/2025-09-08-09-12-15.318383_seed550819/checkpoint_model_000000/model.pt
+training step 0...
+Traceback (most recent call last):
+  File "/home/yitongli/CausVid/causvid/train_ode.py", line 283, in <module>
+    main()
+  File "/home/yitongli/CausVid/causvid/train_ode.py", line 277, in main
+    trainer.train()
+  File "/home/yitongli/CausVid/causvid/train_ode.py", line 241, in train
+    self.generate_video()
+  File "/home/yitongli/CausVid/causvid/train_ode.py", line 213, in generate_video
+    output_path = os.path.join("tmp", f"teacher_{self.step:06d}_{base_name}.mp4")
+UnboundLocalError: local variable 'base_name' referenced before assignment
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/home/yitongli/CausVid/causvid/train_ode.py", line 283, in <module>
+[rank0]:     main()
+[rank0]:   File "/home/yitongli/CausVid/causvid/train_ode.py", line 277, in main
+[rank0]:     trainer.train()
+[rank0]:   File "/home/yitongli/CausVid/causvid/train_ode.py", line 241, in train
+[rank0]:     self.generate_video()
+[rank0]:   File "/home/yitongli/CausVid/causvid/train_ode.py", line 213, in generate_video
+[rank0]:     output_path = os.path.join("tmp", f"teacher_{self.step:06d}_{base_name}.mp4")
+[rank0]: UnboundLocalError: local variable 'base_name' referenced before assignment

output/wandb/run-20250908_091215-tz5j30tc/files/requirements.txt

@@ -0,0 +1,145 @@
+nvidia-cuda-nvrtc-cu12==12.8.93
+filelock==3.19.1
+aiofiles==24.1.0
+PyYAML==6.0.2
+httpx==0.28.1
+multidict==6.6.4
+tifffile==2025.5.10
+tzdata==2025.2
+urllib3==2.5.0
+decord==0.6.0
+certifi==2025.8.3
+setuptools==78.1.1
+websocket-client==1.8.0
+gitdb==4.0.12
+yarl==1.20.1
+tqdm==4.67.1
+open_clip_torch==3.1.0
+pyparsing==3.2.3
+fastapi==0.116.1
+nvidia-curand-cu12==10.3.9.90
+mdurl==0.1.2
+torchvision==0.23.0
+h11==0.16.0
+pytz==2025.2
+six==1.17.0
+nvidia-cufile-cu12==1.13.1.3
+aiohappyeyeballs==2.6.1
+wandb==0.21.3
+nvidia-nvjitlink-cu12==12.8.93
+cycler==0.12.1
+anyio==4.10.0
+scikit-image==0.25.2
+nvidia-cuda-runtime-cu12==12.8.90
+sentencepiece==0.2.1
+rich==14.1.0
+pydantic==2.11.7
+GitPython==3.1.45
+python-dateutil==2.9.0.post0
+accelerate==1.10.1
+proglog==0.1.12
+sentry-sdk==2.37.0
+nvidia-cusparselt-cu12==0.7.1
+dashscope==1.24.3
+platformdirs==4.4.0
+safehttpx==0.1.6
+fsspec==2025.9.0
+lazy_loader==0.4
+typing_extensions==4.15.0
+semantic-version==2.10.0
+numpy==1.24.4
+nvidia-cufft-cu12==11.3.3.83
+ruff==0.12.12
+nvidia-cudnn-cu12==9.10.2.21
+annotated-types==0.7.0
+decorator==4.4.2
+antlr4-python3-runtime==4.9.3
+psutil==7.0.0
+Brotli==1.1.0
+tomlkit==0.13.3
+httpcore==1.0.9
+kiwisolver==1.4.9
+nvidia-cusparse-cu12==12.5.8.93
+groovy==0.1.2
+protobuf==6.32.0
+orjson==3.11.3
+scipy==1.15.3
+regex==2025.9.1
+MarkupSafe==3.0.2
+av==13.1.0
+timm==1.0.19
+uvicorn==0.35.0
+wheel==0.45.1
+Pygments==2.19.2
+websockets==15.0.1
+lmdb==1.7.3
+sympy==1.14.0
+einops==0.8.1
+idna==3.10
+triton==3.4.0
+torch==2.8.0
+moviepy==1.0.3
+nvidia-nvtx-cu12==12.8.90
+matplotlib==3.10.6
+pillow==11.3.0
+charset-normalizer==3.4.3
+attrs==25.3.0
+aiosignal==1.4.0
+markdown-it-py==4.0.0
+requests==2.32.5
+typer==0.17.4
+huggingface-hub==0.34.4
+nvidia-nccl-cu12==2.27.3
+propcache==0.3.2
+opencv-python==4.11.0.86
+ffmpy==0.6.1
+jmespath==1.0.1
+botocore==1.40.25
+pydantic_core==2.33.2
+fonttools==4.59.2
+omegaconf==2.3.0
+pycparser==2.22
+mpmath==1.3.0
+flash_attn==2.8.3
+smmap==5.0.2
+gradio_client==1.12.1
+exceptiongroup==1.3.0
+cffi==1.17.1
+diffusers==0.31.0
+nvidia-cusolver-cu12==11.7.3.90
+contourpy==1.3.2
+cryptography==45.0.7
+nvidia-cuda-cupti-cu12==12.8.90
+wcwidth==0.2.13
+zipp==3.23.0
+safetensors==0.6.2
+gradio==5.44.1
+click==8.2.1
+frozenlist==1.7.0
+networkx==3.4.2
+s3transfer==0.13.1
+shellingham==1.5.4
+starlette==0.47.3
+packaging==25.0
+ftfy==6.3.1
+importlib_metadata==8.7.0
+transformers==4.56.1
+nvidia-cublas-cu12==12.8.4.1
+Jinja2==3.1.6
+pycocotools==2.0.10
+sniffio==1.3.1
+hf-xet==1.1.9
+typing-inspection==0.4.1
+pandas==2.3.2
+python-multipart==0.0.20
+aiohttp==3.12.15
+clip==1.0
+pydub==0.25.1
+easydict==1.13
+pip==25.2
+tokenizers==0.22.0
+imageio==2.37.0
+async-timeout==5.0.1
+boto3==1.40.25
+imageio-ffmpeg==0.6.0
+CausVid==0.0.1

output/wandb/run-20250908_091215-tz5j30tc/files/wandb-metadata.json

@@ -0,0 +1,93 @@
+{
+  "os": "Linux-6.8.0-1028-aws-x86_64-with-glibc2.39",
+  "python": "CPython 3.10.18",
+  "startedAt": "2025-09-08T09:12:15.567002Z",
+  "args": [
+    "--config_path",
+    "configs/wan_causal_ode.yaml"
+  ],
+  "program": "/home/yitongli/CausVid/causvid/train_ode.py",
+  "codePath": "causvid/train_ode.py",
+  "codePathLocal": "causvid/train_ode.py",
+  "git": {
+    "remote": "https://github.com/tianweiy/CausVid.git",
+    "commit": "adb6a5ecd07666b4d0290042915c8406e6d5ce22"
+  },
+  "email": "liyitong.thu@gmail.com",
+  "root": "./output",
+  "host": "ip-172-31-3-169",
+  "executable": "/home/yitongli/miniconda3/envs/causvid/bin/python3.10",
+  "cpu_count": 48,
+  "cpu_count_logical": 96,
+  "gpu": "NVIDIA A100-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "2079114358784",
+      "used": "1400048283648"
+    }
+  },
+  "memory": {
+    "total": "1204521443328"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-88ce35f2-86d5-a394-f90d-a52e2cd45949"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-1039594d-55ae-3eb3-339e-6d4d62c81d42"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-1f47056a-9c2c-8af1-8232-636203622830"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-38de1dff-a680-ec66-625b-49f2a31e3b42"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-712ffb29-5da2-92e5-99b7-9f148e8c1204"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-194ed489-5bfa-defc-85b1-812953e7d84a"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-21b1f0be-4c00-4af7-0649-9942ae8fd0e4"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-195cca5b-c8ee-790b-4d30-ae4d7aa93057"
+    }
+  ],
+  "cudaVersion": "12.8",
+  "writerId": "dzykhs2dp41ohf8z5ba4wgauzbq2dbq2"
+}

output/wandb/run-20250908_091215-tz5j30tc/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_step":0,"_wandb":{"runtime":139},"_runtime":139.90914664,"generator_loss":0.31640625,"generator_grad_norm":5.5625,"loss_at_time_500":0.2705078125,"_timestamp":1.7573228593443248e+09}

output/wandb/run-20250908_091215-tz5j30tc/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2025-09-08T09:12:15.588566306Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpnban0o38/port-4018970.txt","pid":4018970,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-09-08T09:12:15.589212266Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":4018970}
+{"time":"2025-09-08T09:12:15.589223222Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-4018970-4019303-1328136160/socket","Net":"unix"}}
+{"time":"2025-09-08T09:12:15.77464832Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-09-08T09:12:15.785168473Z","level":"INFO","msg":"handleInformInit: received","streamId":"tz5j30tc","id":"1(@)"}
+{"time":"2025-09-08T09:12:16.008129399Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"tz5j30tc","id":"1(@)"}
+{"time":"2025-09-08T09:14:36.117068968Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-09-08T09:14:36.117157346Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-09-08T09:14:36.117145923Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-09-08T09:14:36.117277861Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-4018970-4019303-1328136160/socket","Net":"unix"}}
+{"time":"2025-09-08T09:14:36.117345059Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-09-08T09:14:36.703976989Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-09-08T09:14:36.70401689Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-09-08T09:14:36.7040303Z","level":"INFO","msg":"server is closed"}

output/wandb/run-20250908_091215-tz5j30tc/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2025-09-08T09:12:15.785345022Z","level":"INFO","msg":"stream: starting","core version":"0.21.3"}
+{"time":"2025-09-08T09:12:16.008075812Z","level":"INFO","msg":"stream: created new stream","id":"tz5j30tc"}
+{"time":"2025-09-08T09:12:16.008124642Z","level":"INFO","msg":"stream: started","id":"tz5j30tc"}
+{"time":"2025-09-08T09:12:16.008135566Z","level":"INFO","msg":"handler: started","stream_id":"tz5j30tc"}
+{"time":"2025-09-08T09:12:16.008145832Z","level":"INFO","msg":"sender: started","stream_id":"tz5j30tc"}
+{"time":"2025-09-08T09:12:16.008160919Z","level":"INFO","msg":"writer: started","stream_id":"tz5j30tc"}
+{"time":"2025-09-08T09:14:36.117135291Z","level":"INFO","msg":"stream: closing","id":"tz5j30tc"}
+{"time":"2025-09-08T09:14:36.435658587Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-09-08T09:14:36.58265815Z","level":"INFO","msg":"handler: closed","stream_id":"tz5j30tc"}
+{"time":"2025-09-08T09:14:36.582727241Z","level":"INFO","msg":"sender: closed","stream_id":"tz5j30tc"}
+{"time":"2025-09-08T09:14:36.582737303Z","level":"INFO","msg":"stream: closed","id":"tz5j30tc"}

output/wandb/run-20250908_091215-tz5j30tc/logs/debug.log

@@ -0,0 +1,24 @@
+2025-09-08 09:12:15,568 INFO    MainThread:4018970 [wandb_setup.py:_flush():81] Current SDK version is 0.21.3
+2025-09-08 09:12:15,568 INFO    MainThread:4018970 [wandb_setup.py:_flush():81] Configure stats pid to 4018970
+2025-09-08 09:12:15,568 INFO    MainThread:4018970 [wandb_setup.py:_flush():81] Loading settings from /home/yitongli/.config/wandb/settings
+2025-09-08 09:12:15,568 INFO    MainThread:4018970 [wandb_setup.py:_flush():81] Loading settings from /home/yitongli/CausVid/wandb/settings
+2025-09-08 09:12:15,568 INFO    MainThread:4018970 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2025-09-08 09:12:15,568 INFO    MainThread:4018970 [wandb_init.py:setup_run_log_directory():686] Logging user logs to ./output/wandb/run-20250908_091215-tz5j30tc/logs/debug.log
+2025-09-08 09:12:15,568 INFO    MainThread:4018970 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to ./output/wandb/run-20250908_091215-tz5j30tc/logs/debug-internal.log
+2025-09-08 09:12:15,568 INFO    MainThread:4018970 [wandb_init.py:init():813] calling init triggers
+2025-09-08 09:12:15,568 INFO    MainThread:4018970 [wandb_init.py:init():818] wandb.init called with sweep_config: {}
+config: {'model_name': 'causal_wan', 'generator_grad': {'model': True}, 'denoising_step_list': [1000, 757, 522, 0], 'generator_task': 'causal_video', 'generator_fsdp_wrap_strategy': 'size', 'text_encoder_fsdp_wrap_strategy': 'size', 'mixed_precision': True, 'seed': 550819, 'wandb_host': 'https://api.wandb.ai', 'wandb_key': '5409d3b960b01b25cec0f6abb5361b4022f0cc41', 'wandb_entity': 'liyitong-Tsinghua University', 'wandb_project': 'self-forcing', 'wandb_name': 'wan_causal_ode', 'sharding_strategy': 'hybrid_full', 'lr': 2e-06, 'beta1': 0.9, 'beta2': 0.999, 'data_path': '../mixkit_ode_lmdb', 'batch_size': 2, 'log_iters': 200, 'output_path': './output', 'distillation_loss': 'ode', 'gradient_checkpointing': True, 'num_frame_per_block': 3, 'warp_denoising_step': False, 'no_save': False, '_wandb': {}}
+2025-09-08 09:12:15,568 INFO    MainThread:4018970 [wandb_init.py:init():854] starting backend
+2025-09-08 09:12:15,774 INFO    MainThread:4018970 [wandb_init.py:init():857] sending inform_init request
+2025-09-08 09:12:15,779 INFO    MainThread:4018970 [wandb_init.py:init():865] backend started and connected
+2025-09-08 09:12:15,783 INFO    MainThread:4018970 [wandb_init.py:init():936] updated telemetry
+2025-09-08 09:12:15,788 INFO    MainThread:4018970 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout
+2025-09-08 09:12:16,206 INFO    MainThread:4018970 [wandb_init.py:init():1011] starting run threads in backend
+2025-09-08 09:12:16,315 INFO    MainThread:4018970 [wandb_run.py:_console_start():2494] atexit reg
+2025-09-08 09:12:16,316 INFO    MainThread:4018970 [wandb_run.py:_redirect():2342] redirect: wrap_raw
+2025-09-08 09:12:16,316 INFO    MainThread:4018970 [wandb_run.py:_redirect():2411] Wrapping output streams.
+2025-09-08 09:12:16,316 INFO    MainThread:4018970 [wandb_run.py:_redirect():2434] Redirects installed.
+2025-09-08 09:12:16,318 INFO    MainThread:4018970 [wandb_init.py:init():1057] run started, returning control to user process
+2025-09-08 09:12:17,359 INFO    MainThread:4018970 [wandb_run.py:_config_callback():1380] config_cb None None {'_wandb': {'code_path': 'source-self-forcing-causvid_train_ode.py'}}
+2025-09-08 09:14:36,117 INFO    wandb-AsyncioManager-main:4018970 [service_client.py:_forward_responses():84] Reached EOF.
+2025-09-08 09:14:36,117 INFO    wandb-AsyncioManager-main:4018970 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.

output/wandb/run-20250908_091534-f394z0xa/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2025-09-08T09:15:34.947665584Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpb9cxfvx4/port-4032801.txt","pid":4032801,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-09-08T09:15:34.948269838Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":4032801}
+{"time":"2025-09-08T09:15:34.948238593Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-4032801-4033128-1592522043/socket","Net":"unix"}}
+{"time":"2025-09-08T09:15:35.132762156Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-09-08T09:15:35.13989349Z","level":"INFO","msg":"handleInformInit: received","streamId":"f394z0xa","id":"1(@)"}
+{"time":"2025-09-08T09:15:35.350122319Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"f394z0xa","id":"1(@)"}
+{"time":"2025-09-08T09:17:58.289894961Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-09-08T09:17:58.289957034Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-09-08T09:17:58.289973286Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-09-08T09:17:58.290055209Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-09-08T09:17:58.29008793Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-4032801-4033128-1592522043/socket","Net":"unix"}}
+{"time":"2025-09-08T09:17:58.843012332Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-09-08T09:17:58.843047327Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-09-08T09:17:58.843056791Z","level":"INFO","msg":"server is closed"}

output/wandb/run-20250908_091953-n3vl9u22/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2025-09-08T09:19:53.577274617Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpyl_6c_ow/port-4049443.txt","pid":4049443,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-09-08T09:19:53.577824208Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":4049443}
+{"time":"2025-09-08T09:19:53.577821212Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-4049443-4049733-3298022966/socket","Net":"unix"}}
+{"time":"2025-09-08T09:19:53.762195815Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-09-08T09:19:53.77245292Z","level":"INFO","msg":"handleInformInit: received","streamId":"n3vl9u22","id":"1(@)"}
+{"time":"2025-09-08T09:19:53.97809872Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"n3vl9u22","id":"1(@)"}
+{"time":"2025-09-08T09:31:04.091714838Z","level":"INFO","msg":"server: parent process exited, terminating service process"}

output/wandb/run-20250909_031406-fvhxlznm/files/output.log

@@ -0,0 +1,113 @@
+run dir: ./output/wandb/run-20250909_031406-fvhxlznm/files
+Loading pretrained generator from pretrained_ode.pt
+KV inference with 3 frames per block
+ODERegression initialized.
+ cache a block wise causal mask with block size of 3 frames
+BlockMask(shape=(1, 1, 32768, 32768), sparsity=42.52%,
+(0, 0)
+████░░
+████░░
+████░░░░░░░░
+██████████░░
+██████████░░
+██████████░░░░░░░░
+████████████████░░
+████████████████░░
+████████████████░░░░░░░░
+██████████████████████░░
+██████████████████████░░
+██████████████████████░░░░░░░░
+████████████████████████████░░
+████████████████████████████░░
+████████████████████████████████░░
+████████████████████████████████░░
+████████████████████████████████░░░░░░░░
+████████████████████████████████████████
+████████████████████████████████████████
+████████████████████████████████████████
+)
+Start gathering distributed model states...
+/home/yitongli/miniconda3/envs/causvid/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:678: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Model saved to ./output/2025-09-09-03-14-06.343483_seed6553852/checkpoint_model_000000/model.pt
+training step 0...
+Saving video: 100%|██████████████████████████████████████████████████████████| 81/81 [00:03<00:00, 24.90it/s]
+in main process
+log video_0
+wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes.
+log video_1
+wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes.
+log video_2
+wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes.
+log video_3
+wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes.
+log video_4
+wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes.
+log video_5
+wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes.
+log video_6
+wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes.
+log video_7
+wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes.
+training step 1...
+training step 2...
+training step 3...
+training step 4...
+training step 5...
+training step 6...
+training step 7...
+training step 8...
+training step 9...
+training step 10...
+training step 11...
+training step 12...
+training step 13...
+training step 14...
+training step 15...
+training step 16...
+training step 17...
+training step 18...
+training step 19...
+training step 20...
+training step 21...
+training step 22...
+training step 23...
+training step 24...
+training step 25...
+training step 26...
+training step 27...
+training step 28...
+training step 29...
+training step 30...
+training step 31...
+training step 32...
+training step 33...
+training step 34...
+training step 35...
+training step 36...
+training step 37...
+training step 38...
+training step 39...
+training step 40...
+training step 41...
+training step 42...
+training step 43...
+training step 44...
+training step 45...
+training step 46...
+training step 47...
+training step 48...
+training step 49...
+training step 50...
+training step 51...
+training step 52...
+training step 53...
+training step 54...
+training step 55...
+training step 56...
+training step 57...
+training step 58...
+training step 59...
+training step 60...
+training step 61...
+training step 62...

output/wandb/run-20250909_031406-fvhxlznm/files/requirements.txt

@@ -0,0 +1,145 @@
+nvidia-cuda-nvrtc-cu12==12.8.93
+filelock==3.19.1
+aiofiles==24.1.0
+PyYAML==6.0.2
+httpx==0.28.1
+multidict==6.6.4
+tifffile==2025.5.10
+tzdata==2025.2
+urllib3==2.5.0
+decord==0.6.0
+certifi==2025.8.3
+setuptools==78.1.1
+websocket-client==1.8.0
+gitdb==4.0.12
+yarl==1.20.1
+tqdm==4.67.1
+open_clip_torch==3.1.0
+pyparsing==3.2.3
+fastapi==0.116.1
+nvidia-curand-cu12==10.3.9.90
+mdurl==0.1.2
+torchvision==0.23.0
+h11==0.16.0
+pytz==2025.2
+six==1.17.0
+nvidia-cufile-cu12==1.13.1.3
+aiohappyeyeballs==2.6.1
+wandb==0.21.3
+nvidia-nvjitlink-cu12==12.8.93
+cycler==0.12.1
+anyio==4.10.0
+scikit-image==0.25.2
+nvidia-cuda-runtime-cu12==12.8.90
+sentencepiece==0.2.1
+rich==14.1.0
+pydantic==2.11.7
+GitPython==3.1.45
+python-dateutil==2.9.0.post0
+accelerate==1.10.1
+proglog==0.1.12
+sentry-sdk==2.37.0
+nvidia-cusparselt-cu12==0.7.1
+dashscope==1.24.3
+platformdirs==4.4.0
+safehttpx==0.1.6
+fsspec==2025.9.0
+lazy_loader==0.4
+typing_extensions==4.15.0
+semantic-version==2.10.0
+numpy==1.24.4
+nvidia-cufft-cu12==11.3.3.83
+ruff==0.12.12
+nvidia-cudnn-cu12==9.10.2.21
+annotated-types==0.7.0
+decorator==4.4.2
+antlr4-python3-runtime==4.9.3
+psutil==7.0.0
+Brotli==1.1.0
+tomlkit==0.13.3
+httpcore==1.0.9
+kiwisolver==1.4.9
+nvidia-cusparse-cu12==12.5.8.93
+groovy==0.1.2
+protobuf==6.32.0
+orjson==3.11.3
+scipy==1.15.3
+regex==2025.9.1
+MarkupSafe==3.0.2
+av==13.1.0
+timm==1.0.19
+uvicorn==0.35.0
+wheel==0.45.1
+Pygments==2.19.2
+websockets==15.0.1
+lmdb==1.7.3
+sympy==1.14.0
+einops==0.8.1
+idna==3.10
+triton==3.4.0
+torch==2.8.0
+moviepy==1.0.3
+nvidia-nvtx-cu12==12.8.90
+matplotlib==3.10.6
+pillow==11.3.0
+charset-normalizer==3.4.3
+attrs==25.3.0
+aiosignal==1.4.0
+markdown-it-py==4.0.0
+requests==2.32.5
+typer==0.17.4
+huggingface-hub==0.34.4
+nvidia-nccl-cu12==2.27.3
+propcache==0.3.2
+opencv-python==4.11.0.86
+ffmpy==0.6.1
+jmespath==1.0.1
+botocore==1.40.25
+pydantic_core==2.33.2
+fonttools==4.59.2
+omegaconf==2.3.0
+pycparser==2.22
+mpmath==1.3.0
+flash_attn==2.8.3
+smmap==5.0.2
+gradio_client==1.12.1
+exceptiongroup==1.3.0
+cffi==1.17.1
+diffusers==0.31.0
+nvidia-cusolver-cu12==11.7.3.90
+contourpy==1.3.2
+cryptography==45.0.7
+nvidia-cuda-cupti-cu12==12.8.90
+wcwidth==0.2.13
+zipp==3.23.0
+safetensors==0.6.2
+gradio==5.44.1
+click==8.2.1
+frozenlist==1.7.0
+networkx==3.4.2
+s3transfer==0.13.1
+shellingham==1.5.4
+starlette==0.47.3
+packaging==25.0
+ftfy==6.3.1
+importlib_metadata==8.7.0
+transformers==4.56.1
+nvidia-cublas-cu12==12.8.4.1
+Jinja2==3.1.6
+pycocotools==2.0.10
+sniffio==1.3.1
+hf-xet==1.1.9
+typing-inspection==0.4.1
+pandas==2.3.2
+python-multipart==0.0.20
+aiohttp==3.12.15
+clip==1.0
+pydub==0.25.1
+easydict==1.13
+pip==25.2
+tokenizers==0.22.0
+imageio==2.37.0
+async-timeout==5.0.1
+boto3==1.40.25
+imageio-ffmpeg==0.6.0
+CausVid==0.0.1

output/wandb/run-20250909_031406-fvhxlznm/files/wandb-metadata.json

@@ -0,0 +1,93 @@
+{
+  "os": "Linux-6.8.0-1028-aws-x86_64-with-glibc2.39",
+  "python": "CPython 3.10.18",
+  "startedAt": "2025-09-09T03:14:06.607859Z",
+  "args": [
+    "--config_path",
+    "configs/wan_causal_ode.yaml"
+  ],
+  "program": "/home/yitongli/CausVid/causvid/train_ode.py",
+  "codePath": "causvid/train_ode.py",
+  "codePathLocal": "causvid/train_ode.py",
+  "git": {
+    "remote": "https://github.com/tianweiy/CausVid.git",
+    "commit": "adb6a5ecd07666b4d0290042915c8406e6d5ce22"
+  },
+  "email": "liyitong.thu@gmail.com",
+  "root": "./output",
+  "host": "ip-172-31-3-169",
+  "executable": "/home/yitongli/miniconda3/envs/causvid/bin/python3.10",
+  "cpu_count": 48,
+  "cpu_count_logical": 96,
+  "gpu": "NVIDIA A100-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "2079114358784",
+      "used": "1450179067904"
+    }
+  },
+  "memory": {
+    "total": "1204521443328"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-88ce35f2-86d5-a394-f90d-a52e2cd45949"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-1039594d-55ae-3eb3-339e-6d4d62c81d42"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-1f47056a-9c2c-8af1-8232-636203622830"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-38de1dff-a680-ec66-625b-49f2a31e3b42"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-712ffb29-5da2-92e5-99b7-9f148e8c1204"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-194ed489-5bfa-defc-85b1-812953e7d84a"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-21b1f0be-4c00-4af7-0649-9942ae8fd0e4"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-195cca5b-c8ee-790b-4d30-ae4d7aa93057"
+    }
+  ],
+  "cudaVersion": "12.8",
+  "writerId": "fsrpwg7v4b4972ihmv15clw215fcozrh"
+}

output/wandb/run-20250909_031406-fvhxlznm/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2025-09-09T03:14:06.646254908Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp527vay8l/port-1234529.txt","pid":1234529,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-09-09T03:14:06.646757368Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1234529}
+{"time":"2025-09-09T03:14:06.646738738Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1234529-1235338-3412218744/socket","Net":"unix"}}
+{"time":"2025-09-09T03:14:06.816587266Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-09-09T03:14:06.827214814Z","level":"INFO","msg":"handleInformInit: received","streamId":"fvhxlznm","id":"1(@)"}
+{"time":"2025-09-09T03:14:07.041711266Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"fvhxlznm","id":"1(@)"}
+{"time":"2025-09-09T04:05:19.585076497Z","level":"INFO","msg":"server: parent process exited, terminating service process"}

output/wandb/run-20250909_031406-fvhxlznm/logs/debug-internal.log

@@ -0,0 +1,6 @@
+{"time":"2025-09-09T03:14:06.827346748Z","level":"INFO","msg":"stream: starting","core version":"0.21.3"}
+{"time":"2025-09-09T03:14:07.041670446Z","level":"INFO","msg":"stream: created new stream","id":"fvhxlznm"}
+{"time":"2025-09-09T03:14:07.04170638Z","level":"INFO","msg":"stream: started","id":"fvhxlznm"}
+{"time":"2025-09-09T03:14:07.04171651Z","level":"INFO","msg":"writer: started","stream_id":"fvhxlznm"}
+{"time":"2025-09-09T03:14:07.041726675Z","level":"INFO","msg":"handler: started","stream_id":"fvhxlznm"}
+{"time":"2025-09-09T03:14:07.041739753Z","level":"INFO","msg":"sender: started","stream_id":"fvhxlznm"}

output/wandb/run-20250909_031406-fvhxlznm/logs/debug.log

@@ -0,0 +1,22 @@
+2025-09-09 03:14:06,609 INFO    MainThread:1234529 [wandb_setup.py:_flush():81] Current SDK version is 0.21.3
+2025-09-09 03:14:06,609 INFO    MainThread:1234529 [wandb_setup.py:_flush():81] Configure stats pid to 1234529
+2025-09-09 03:14:06,609 INFO    MainThread:1234529 [wandb_setup.py:_flush():81] Loading settings from /home/yitongli/.config/wandb/settings
+2025-09-09 03:14:06,609 INFO    MainThread:1234529 [wandb_setup.py:_flush():81] Loading settings from /home/yitongli/CausVid/wandb/settings
+2025-09-09 03:14:06,609 INFO    MainThread:1234529 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2025-09-09 03:14:06,609 INFO    MainThread:1234529 [wandb_init.py:setup_run_log_directory():686] Logging user logs to ./output/wandb/run-20250909_031406-fvhxlznm/logs/debug.log
+2025-09-09 03:14:06,609 INFO    MainThread:1234529 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to ./output/wandb/run-20250909_031406-fvhxlznm/logs/debug-internal.log
+2025-09-09 03:14:06,609 INFO    MainThread:1234529 [wandb_init.py:init():813] calling init triggers
+2025-09-09 03:14:06,609 INFO    MainThread:1234529 [wandb_init.py:init():818] wandb.init called with sweep_config: {}
+config: {'model_name': 'causal_wan', 'generator_ckpt': 'pretrained_ode.pt', 'generator_grad': {'model': True}, 'denoising_step_list': [1000, 757, 522, 0], 'generator_task': 'causal_video', 'generator_fsdp_wrap_strategy': 'size', 'text_encoder_fsdp_wrap_strategy': 'size', 'mixed_precision': True, 'seed': 6553852, 'wandb_host': 'https://api.wandb.ai', 'wandb_key': '5409d3b960b01b25cec0f6abb5361b4022f0cc41', 'wandb_entity': 'liyitong-Tsinghua University', 'wandb_project': 'self-forcing', 'wandb_name': 'wan_causal_ode', 'sharding_strategy': 'hybrid_full', 'lr': 2e-06, 'beta1': 0.9, 'beta2': 0.999, 'data_path': '../mixkit_ode_lmdb', 'batch_size': 2, 'log_iters': 200, 'output_path': './output', 'distillation_loss': 'ode', 'gradient_checkpointing': True, 'num_frame_per_block': 3, 'warp_denoising_step': False, 'no_save': False, '_wandb': {}}
+2025-09-09 03:14:06,609 INFO    MainThread:1234529 [wandb_init.py:init():854] starting backend
+2025-09-09 03:14:06,816 INFO    MainThread:1234529 [wandb_init.py:init():857] sending inform_init request
+2025-09-09 03:14:06,821 INFO    MainThread:1234529 [wandb_init.py:init():865] backend started and connected
+2025-09-09 03:14:06,825 INFO    MainThread:1234529 [wandb_init.py:init():936] updated telemetry
+2025-09-09 03:14:06,830 INFO    MainThread:1234529 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout
+2025-09-09 03:14:07,304 INFO    MainThread:1234529 [wandb_init.py:init():1011] starting run threads in backend
+2025-09-09 03:14:07,415 INFO    MainThread:1234529 [wandb_run.py:_console_start():2494] atexit reg
+2025-09-09 03:14:07,415 INFO    MainThread:1234529 [wandb_run.py:_redirect():2342] redirect: wrap_raw
+2025-09-09 03:14:07,415 INFO    MainThread:1234529 [wandb_run.py:_redirect():2411] Wrapping output streams.
+2025-09-09 03:14:07,415 INFO    MainThread:1234529 [wandb_run.py:_redirect():2434] Redirects installed.
+2025-09-09 03:14:07,417 INFO    MainThread:1234529 [wandb_init.py:init():1057] run started, returning control to user process
+2025-09-09 03:14:07,902 INFO    MainThread:1234529 [wandb_run.py:_config_callback():1380] config_cb None None {'_wandb': {'code_path': 'source-self-forcing-causvid_train_ode.py'}}

video_processing.log

@@ -0,0 +1,2 @@
+2025-09-08 05:00:38,647 - INFO - Batch processing completed in 0.00 seconds
+2025-09-08 05:01:15,712 - INFO - Batch processing completed in 0.00 seconds