Spaces:
Sleeping
Sleeping
File size: 2,824 Bytes
bb58af7 a864c4e bb58af7 a864c4e bb58af7 a864c4e bb58af7 a864c4e bb58af7 a864c4e bb58af7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 | from typing import List
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_core.documents import Document
import PyPDF2
from docx import Document as DocxDocument
class DocumentProcessor:
def __init__(self, chunk_size: int = 1000, chunk_overlap: int = 200):
"""
Initialize document processor with text splitting configuration.
Args:
chunk_size: Maximum characters per chunk (default: 1000)
chunk_overlap: Characters to overlap between chunks (default: 200)
"""
self.text_splitter = RecursiveCharacterTextSplitter(
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
length_function=len,
)
def _chunk_text(self, file_path: str, text: str, doc_type: str) -> List[Document]:
"""
Split text into overlapping chunks with metadata for better retrieval.
Args:
file_path: Original file path for metadata
text: Text content to split
doc_type: Document type (pdf/docx/txt)
Returns:
List[Document]: Chunked documents with metadata
"""
# Create documents with metadata
return self.text_splitter.create_documents(
[text],
metadatas=[{"source": file_path, "type": doc_type}],
)
def process_pdf(self, file_path: str) -> List[Document]:
"""
Extract text from PDF file and convert to chunked documents.
Args:
file_path: Path to PDF file
Returns:
List[Document]: Processed document chunks
"""
reader = PyPDF2.PdfReader(file_path)
text = ""
for page_num, page in enumerate(reader.pages):
page_text = page.extract_text()
if page_text:
text += f"\n---- Page {page_num + 1} ----\n{page_text}"
return self._chunk_text(file_path, text, "pdf")
def process_docx(self, file_path: str) -> List[Document]:
"""
Extract text from DOCX file and convert to chunked documents.
Args:
file_path: Path to DOCX file
Returns:
List[Document]: Processed document chunks
"""
doc = DocxDocument(file_path)
text = "\n".join([paragraph.text for paragraph in doc.paragraphs])
return self._chunk_text(file_path, text, "docx")
def process_txt(self, file_path: str) -> List[Document]:
"""
Read text file and convert to chunked documents.
Args:
file_path: Path to TXT file
Returns:
List[Document]: Processed document chunks
"""
with open(file_path, "r", encoding="utf-8") as file:
text = file.read()
return self._chunk_text(file_path, text, "txt")
|