fix: Sync audio with message timing using CompositeAudioClip

modules/text_story/services/video_composer.py

@@ -8,8 +8,8 @@ import logging
 from typing import List, Dict, Optional
 from moviepy.editor import (
     VideoFileClip, ImageClip, AudioFileClip, 
-    CompositeVideoClip, concatenate_videoclips,
-    ColorClip, TextClip
+    CompositeVideoClip, CompositeAudioClip, concatenate_videoclips,
+    ColorClip, TextClip, afx
 )
 from moviepy.video.fx.all import fadein, fadeout
 import numpy as np
@@ -101,27 +101,10 @@ class VideoComposer:
         """
         logger.info(f"VideoComposer: Starting composition with {len(messages)} messages")
         
-        # Calculate total duration
+        # Calculate total duration and audio start times
         total_duration = 0
-        for i, msg in enumerate(messages):
-            total_duration += self._calculate_typing_duration(msg["text"])
-            total_duration += TIMING["voice_delay"]
-            total_duration += msg["duration"]
-            total_duration += TIMING["micro_silence"]
-        
-        total_duration += TIMING["last_msg_pause"]
-        
-        if ending_text:
-            total_duration += TIMING["ending_duration"]
-        
-        logger.info(f"VideoComposer: Total duration: {total_duration:.1f}s")
-        
-        # Load background
-        background = self.bg_handler.load_and_process(total_duration)
-        
-        # Create message sequence
-        clips = []
-        current_time = 0
+        audio_clips = []  # List of (audio_clip, start_time)
+        frame_sequence = []  # List of (frame, duration, start_time)
         displayed_messages = []
         
         for i, msg in enumerate(messages):
@@ -130,60 +113,76 @@ class VideoComposer:
             # 1. Typing indicator phase
             typing_duration = self._calculate_typing_duration(msg["text"])
             typing_frame = self.renderer.render_frame(displayed_messages, show_typing=True)
-            typing_clip = self._pil_to_moviepy(typing_frame, typing_duration)
-            typing_clip = typing_clip.set_start(current_time)
-            clips.append(typing_clip)
-            current_time += typing_duration
+            frame_sequence.append((typing_frame, typing_duration, total_duration))
+            total_duration += typing_duration
             
-            # 2. Add message (with voice delay)
+            # 2. Add message to display
             displayed_messages.append(msg_dict)
             
-            # Message appears (voice delay gap)
+            # 3. Voice delay (message appears, but audio hasn't started)
             msg_frame = self.renderer.render_frame(displayed_messages)
-            voice_delay_clip = self._pil_to_moviepy(msg_frame, TIMING["voice_delay"])
-            voice_delay_clip = voice_delay_clip.set_start(current_time)
-            clips.append(voice_delay_clip)
-            current_time += TIMING["voice_delay"]
+            frame_sequence.append((msg_frame, TIMING["voice_delay"], total_duration))
+            total_duration += TIMING["voice_delay"]
             
-            # 3. Message with audio
-            msg_clip = self._pil_to_moviepy(msg_frame, msg["duration"])
-            msg_clip = msg_clip.set_start(current_time)
+            # 4. Audio starts HERE - sync point!
+            audio_start_time = total_duration
+            logger.info(f"VideoComposer: Message {i+1} audio starts at {audio_start_time:.2f}s")
             
-            # Add audio
+            # Load audio and add to list with start time
             audio = AudioFileClip(msg["path"])
-            msg_clip = msg_clip.set_audio(audio.set_start(0))
-            clips.append(msg_clip)
-            current_time += msg["duration"]
+            audio_clips.append((audio, audio_start_time))
             
-            # 4. Micro silence
-            silence_clip = self._pil_to_moviepy(msg_frame, TIMING["micro_silence"])
-            silence_clip = silence_clip.set_start(current_time)
-            clips.append(silence_clip)
-            current_time += TIMING["micro_silence"]
+            # Message display during audio
+            frame_sequence.append((msg_frame, msg["duration"], total_duration))
+            total_duration += msg["duration"]
             
-            logger.info(f"VideoComposer: Message {i+1}/{len(messages)} at {current_time:.1f}s")
+            # 5. Micro silence after message
+            frame_sequence.append((msg_frame, TIMING["micro_silence"], total_duration))
+            total_duration += TIMING["micro_silence"]
         
         # Last message pause
         final_frame = self.renderer.render_frame(displayed_messages)
-        pause_clip = self._pil_to_moviepy(final_frame, TIMING["last_msg_pause"])
-        pause_clip = pause_clip.set_start(current_time)
-        clips.append(pause_clip)
-        current_time += TIMING["last_msg_pause"]
+        frame_sequence.append((final_frame, TIMING["last_msg_pause"], total_duration))
+        total_duration += TIMING["last_msg_pause"]
         
         # Ending text
         if ending_text:
-            ending_clip = self._create_ending_clip(ending_text, displayed_messages)
-            ending_clip = ending_clip.set_start(current_time)
-            clips.append(ending_clip)
+            ending_frame = self._create_ending_frame(ending_text, displayed_messages)
+            frame_sequence.append((ending_frame, TIMING["ending_duration"], total_duration))
+            total_duration += TIMING["ending_duration"]
+        
+        logger.info(f"VideoComposer: Total duration: {total_duration:.1f}s")
+        
+        # Load background for full duration
+        background = self.bg_handler.load_and_process(total_duration)
+        
+        # Create video clips from frames
+        video_clips = [background]
+        for frame, duration, start_time in frame_sequence:
+            clip = self._pil_to_moviepy(frame, duration)
+            clip = clip.set_start(start_time)
+            video_clips.append(clip)
+        
+        # Create composite audio with correct start times
+        audio_with_timing = []
+        for audio, start_time in audio_clips:
+            audio_with_timing.append(audio.set_start(start_time))
+        
+        # Composite video
+        video = CompositeVideoClip(video_clips, size=(CANVAS_WIDTH, CANVAS_HEIGHT))
+        video = video.set_duration(total_duration)
         
-        # Composite all clips over background
-        final = CompositeVideoClip([background] + clips, size=(CANVAS_WIDTH, CANVAS_HEIGHT))
-        final = final.set_duration(total_duration)
+        # Composite audio
+        if audio_with_timing:
+            final_audio = CompositeAudioClip(audio_with_timing)
+            video = video.set_audio(final_audio)
         
         # Export
         output_path = os.path.join(output_dir, "text_story_output.mp4")
         
-        final.write_videofile(
+        logger.info(f"VideoComposer: Writing video to {output_path}")
+        
+        video.write_videofile(
             output_path,
             fps=30,
             codec="libx264",
@@ -194,16 +193,16 @@ class VideoComposer:
         )
         
         # Cleanup
-        final.close()
+        video.close()
         background.close()
-        for clip in clips:
-            clip.close()
+        for audio, _ in audio_clips:
+            audio.close()
         
         logger.info(f"VideoComposer: Saved to {output_path}")
         return output_path
     
-    def _create_ending_clip(self, text: str, messages: List[dict]) -> ImageClip:
-        """Create ending text overlay."""
+    def _create_ending_frame(self, text: str, messages: List[dict]) -> Image.Image:
+        """Create ending text overlay frame."""
         # Render current state
         frame = self.renderer.render_frame(messages)
         
@@ -229,8 +228,4 @@ class VideoComposer:
         draw.text((x + 2, y + 2), text, fill=(0, 0, 0, 200), font=font)
         draw.text((x, y), text, fill=(255, 255, 255, 255), font=font)
         
-        # Convert to clip with fade
-        clip = self._pil_to_moviepy(frame, TIMING["ending_duration"])
-        clip = clip.fx(fadein, TIMING["ending_fade"])
-        
-        return clip
+        return frame