Update app.py

app.py

@@ -41,6 +41,7 @@ import spaces
 import gradio as gr
 import numpy as np
 from huggingface_hub import hf_hub_download, snapshot_download
+from ltx_core.loader import LoraPathStrengthAndSDOps, LTXV_LORA_COMFY_RENAMING_MAP
 
 from ltx_core.components.diffusion_steps import EulerDiffusionStep
 from ltx_core.components.noisers import GaussianNoiser
@@ -74,6 +75,8 @@ except Exception as e:
 
 logging.getLogger().setLevel(logging.INFO)
 
+LORA_RUNTIME_SCALE = 1.0
+
 MAX_SEED = np.iinfo(np.int32).max
 DEFAULT_PROMPT = (
     "An astronaut hatches from a fragile egg on the surface of the Moon, "
@@ -267,6 +270,11 @@ checkpoint_path = hf_hub_download(repo_id=LTX_MODEL_REPO, filename="ltx-2.3-22b-
 spatial_upsampler_path = hf_hub_download(repo_id=LTX_MODEL_REPO, filename="ltx-2.3-spatial-upscaler-x2-1.0.safetensors")
 gemma_root = snapshot_download(repo_id=GEMMA_REPO)
 
+lora_path = hf_hub_download(
+    repo_id="dagloop5/LoRA",
+    filename="LoRA2.safetensors"
+)
+
 print(f"Checkpoint: {checkpoint_path}")
 print(f"Spatial upsampler: {spatial_upsampler_path}")
 print(f"Gemma root: {gemma_root}")
@@ -276,7 +284,13 @@ pipeline = LTX23DistilledA2VPipeline(
     distilled_checkpoint_path=checkpoint_path,
     spatial_upsampler_path=spatial_upsampler_path,
     gemma_root=gemma_root,
-    loras=[],
+    loras=[
+        LoraPathStrengthAndSDOps(
+            lora_path,
+            1.0,  # fixed internal strength
+            LTXV_LORA_COMFY_RENAMING_MAP
+        )
+    ],
     quantization=QuantizationPolicy.fp8_cast(),
 )
 
@@ -284,6 +298,20 @@ pipeline = LTX23DistilledA2VPipeline(
 print("Preloading all models (including Gemma and audio components)...")
 ledger = pipeline.model_ledger
 _transformer = ledger.transformer()
+_original_forward = _transformer.forward
+
+def _lora_scaled_forward(*args, **kwargs):
+    out = _original_forward(*args, **kwargs)
+
+    # Apply runtime scaling to LoRA-influenced output
+    # (LTX merges LoRA into attention residuals, so we scale output delta)
+    if isinstance(out, tuple):
+        return tuple(o * LORA_RUNTIME_SCALE if torch.is_tensor(o) else o for o in out)
+    elif torch.is_tensor(out):
+        return out * LORA_RUNTIME_SCALE
+    return out
+
+_transformer.forward = _lora_scaled_forward
 _video_encoder = ledger.video_encoder()
 _video_decoder = ledger.video_decoder()
 _audio_encoder = ledger.audio_encoder()
@@ -355,6 +383,7 @@ def generate_video(
     input_audio,
     prompt: str,
     duration: float,
+    lora_strength: float
     enhance_prompt: bool = True,
     seed: int = 42,
     randomize_seed: bool = True,
@@ -367,6 +396,8 @@ def generate_video(
         log_memory("start")
 
         current_seed = random.randint(0, MAX_SEED) if randomize_seed else int(seed)
+        global LORA_RUNTIME_SCALE
+        LORA_RUNTIME_SCALE = lora_strength
 
         frame_rate = DEFAULT_FRAME_RATE
         num_frames = int(duration * frame_rate) + 1
@@ -451,6 +482,13 @@ with gr.Blocks(title="LTX-2.3 Heretic Distilled") as demo:
                 placeholder="Describe the motion and animation you want...",
             )
             duration = gr.Slider(label="Duration (seconds)", minimum=1.0, maximum=10.0, value=3.0, step=0.1)
+            lora_strength = gr.Slider(
+                label="LoRA Strength",
+                minimum=0.0,
+                maximum=1.5,
+                value=1.0,
+                step=0.05,
+            )
                 
 
             generate_btn = gr.Button("Generate Video", variant="primary", size="lg")
@@ -515,7 +553,7 @@ with gr.Blocks(title="LTX-2.3 Heretic Distilled") as demo:
     generate_btn.click(
         fn=generate_video,
         inputs=[
-            first_image, last_image, input_audio, prompt, duration, enhance_prompt,
+            first_image, last_image, input_audio, prompt, duration, lora_strength, enhance_prompt,
             seed, randomize_seed, height, width,
         ],
         outputs=[output_video, seed],