Update app.py

app.py

@@ -5,15 +5,15 @@ from langchain.document_loaders import PyPDFLoader, Docx2txtLoader, TextLoader
 from langchain.text_splitter import CharacterTextSplitter
 from langchain.vectorstores import Chroma
 from langchain.embeddings import HuggingFaceEmbeddings
-from langchain.llms import HuggingFacePipeline
-from transformers import pipeline
+from langchain.llms.base import LLM
+from huggingface_hub import InferenceClient
 import gradio as gr
 
 # workaround for sqlite in HF spaces
 __import__('pysqlite3')
 sys.modules['sqlite3'] = sys.modules.pop('pysqlite3')
 
-# 📄 Load documents from multiple_docs
+# 📄 Load documents
 docs = []
 for f in os.listdir("multiple_docs"):
     if f.endswith(".pdf"):
@@ -26,11 +26,10 @@ for f in os.listdir("multiple_docs"):
         loader = TextLoader(os.path.join("multiple_docs", f))
         docs.extend(loader.load())
 
-# 🔗 Split into smaller chunks
+# 🔗 Split into chunks
 splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=10)
 docs = splitter.split_documents(docs)
 
-# 🧬 Prepare texts and metadata
 texts = [doc.page_content for doc in docs]
 metadatas = [{"id": i} for i in range(len(texts))]
 
@@ -45,12 +44,39 @@ vectorstore = Chroma(
 vectorstore.add_texts(texts=texts, metadatas=metadatas)
 vectorstore.persist()
 
-# 🤖 Load free LLM using pipeline + wrap in HuggingFacePipeline
-model_name = "google/flan-t5-large"  # or flan-t5-base for faster
-hf_pipeline = pipeline("text2text-generation", model=model_name, device=-1)  # CPU
-llm = HuggingFacePipeline(pipeline=hf_pipeline)
+# 🔐 Get HF token from env variable
+HF_API_KEY = os.getenv("HF_API_KEY")
+if HF_API_KEY is None:
+    raise ValueError("HUGGINGFACE_API_KEY environment variable is not set.")
 
-# 🔗 Create conversational chain
+# 🤖 Use HuggingFace Inference API (cloud) for LLM
+HF_MODEL = "deepseek-ai/deepseek-llm-7b-instruct"  # you can change this to another model if you like
+client = InferenceClient(token=HF_API_KEY)
+
+# 🔷 Wrap HF client into LangChain LLM interface
+class HuggingFaceInferenceLLM(LLM):
+    """LLM that queries HuggingFace Inference API."""
+
+    model: str = HF_MODEL
+    client: InferenceClient = client
+
+    def _call(self, prompt, stop=None, run_manager=None, **kwargs):
+        response = self.client.text_generation(
+            model=self.model,
+            inputs=prompt,
+            max_new_tokens=512,
+            temperature=0.7,
+            do_sample=True,
+        )
+        return response
+
+    @property
+    def _llm_type(self) -> str:
+        return "huggingface_inference_api"
+
+llm = HuggingFaceInferenceLLM()
+
+# 🔗 Conversational chain
 chain = ConversationalRetrievalChain.from_llm(
     llm,
     retriever=vectorstore.as_retriever(search_kwargs={'k': 6}),