src/analysis/analysis_cleaner.py

import re

class AnalysisCleaner:
    def __init__(self):
        self.seen_paragraphs = set()

    def remove_duplicates(self, text: str) -> str:
        """Remove duplicate paragraphs while preserving order"""
        paragraphs = text.split('\n\n')
        unique_paragraphs = []

        for paragraph in paragraphs:
            # Create a normalized version for comparison
            normalized = ' '.join(paragraph.lower().split())
            if normalized and normalized not in self.seen_paragraphs:
                self.seen_paragraphs.add(normalized)
                unique_paragraphs.append(paragraph)

        return '\n\n'.join(unique_paragraphs)

    def reorganize_content(self, text: str) -> str:
        """Convert bullet points into flowing paragraphs"""
        lines = text.split('\n')
        current_paragraph = []
        flowing_text = []

        for line in lines:
            # Remove bullet points and numbering
            cleaned_line = re.sub(r'^\s*[\*\-\•]\s*', '', line)
            cleaned_line = re.sub(r'^\s*\d+\.\s*', '', cleaned_line)

            if cleaned_line.strip():
                if cleaned_line.startswith('###'):  # Keep section headers
                    if current_paragraph:
                        flowing_text.append(' '.join(current_paragraph))
                        current_paragraph = []
                    flowing_text.append(cleaned_line)
                else:
                    current_paragraph.append(cleaned_line)
            elif current_paragraph:
                flowing_text.append(' '.join(current_paragraph))
                current_paragraph = []

        if current_paragraph:
            flowing_text.append(' '.join(current_paragraph))

        return '\n\n'.join(flowing_text)

    def clean_analysis(self, text: str) -> str:
        """Apply all cleanup steps"""
        # Remove duplicate content 
        cleaned = self.remove_duplicates(text)

        # Convert to flowing paragraphs
        cleaned = self.reorganize_content(cleaned)

        # Clean up extra whitespace
        cleaned = re.sub(r'\n{3,}', '\n\n', cleaned)

        return cleaned