[bugfix]fix the cut-off issue due to LLM predict token limit(256 for openai python lib default), by setting temperature to 0 and set LLM predict method from compact-refine to refine

.gitignore

@@ -56,6 +56,7 @@ coverage.xml
 .hypothesis/
 .pytest_cache/
 .ruff_cache
+wandb/
 
 # Translations
 *.mo

app.py

@@ -9,9 +9,9 @@ from xpipe_wiki.manager_factory import XPipeRobotManagerFactory, XPipeRobotRevis
 logging.basicConfig(
     stream=sys.stdout, level=logging.INFO
 )  # logging.DEBUG for more verbose output
-logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))
+# logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))
 
-# Sidebar contents
+# # Sidebar contents
 with st.sidebar:
     st.title("🤗💬 LLM Chat App")
     st.markdown(
@@ -29,8 +29,9 @@ with st.sidebar:
 
 def main() -> None:
     st.header("X-Pipe Wiki 机器人 💬")
+
     robot_manager = XPipeRobotManagerFactory.get_or_create(
-        XPipeRobotRevision.HUGGINGFACE_VERSION_0
+        XPipeRobotRevision.SIMPLE_OPENAI_VERSION_0
     )
     robot = robot_manager.get_robot()
     query = st.text_input("X-Pipe Wiki 问题:")

core/test_lifecycle.py

@@ -1,10 +1,7 @@
-import logging
 from unittest import TestCase
 
 from core.lifecycle import Lifecycle
 
-logging.basicConfig()
-
 
 class SubLifecycle(Lifecycle):
     def __init__(self) -> None:

dataset/index_store.json

@@ -1 +1 @@
-{"index_store/data": {"f0aa9b38-a1b4-4cd8-8261-38f7e28b7053": {"__type__": "vector_store", "__data__": "{\"index_id\": \"f0aa9b38-a1b4-4cd8-8261-38f7e28b7053\", \"summary\": null, \"nodes_dict\": {\"432d9349-9c31-4ff4-b982-7a3d80a383a4\": \"432d9349-9c31-4ff4-b982-7a3d80a383a4\", \"1c93f814-8019-43d5-a583-12e1c1c1d702\": \"1c93f814-8019-43d5-a583-12e1c1c1d702\", \"6850a6e1-7bf9-42c8-873c-fff410bf4f61\": \"6850a6e1-7bf9-42c8-873c-fff410bf4f61\", \"99678815-82b6-4df5-88e5-8f489ef5f0dd\": \"99678815-82b6-4df5-88e5-8f489ef5f0dd\", \"f56f6064-9d14-4290-bfb5-b5992fddbfd8\": \"f56f6064-9d14-4290-bfb5-b5992fddbfd8\", \"d8cd4c80-4fcd-4961-a6f9-e315078bdd3b\": \"d8cd4c80-4fcd-4961-a6f9-e315078bdd3b\", \"3b4b0e7f-dcde-4cf2-94a5-e96fa67d52bb\": \"3b4b0e7f-dcde-4cf2-94a5-e96fa67d52bb\", \"1a275084-07c0-41cb-85d5-20b454630cde\": \"1a275084-07c0-41cb-85d5-20b454630cde\", \"0fdf0b9e-db8c-46d3-a634-67bd4b3939ca\": \"0fdf0b9e-db8c-46d3-a634-67bd4b3939ca\", \"a428ca4f-d82d-4dbd-98ef-63dce24a749d\": \"a428ca4f-d82d-4dbd-98ef-63dce24a749d\", \"1f537ab9-d15c-456a-937e-481b1b84dfd1\": \"1f537ab9-d15c-456a-937e-481b1b84dfd1\", \"598cd507-a5c5-4b6d-a555-037d1fc388d5\": \"598cd507-a5c5-4b6d-a555-037d1fc388d5\", \"65f05413-df67-4a93-80cf-83654439fdd8\": \"65f05413-df67-4a93-80cf-83654439fdd8\", \"dc8cbec7-d6e5-462c-977a-738f1eedcffc\": \"dc8cbec7-d6e5-462c-977a-738f1eedcffc\", \"a5771b3f-00f7-4669-b593-647827e570c8\": \"a5771b3f-00f7-4669-b593-647827e570c8\", \"cc945889-e48c-4262-9654-86d60bfe38b6\": \"cc945889-e48c-4262-9654-86d60bfe38b6\", \"1e6cc8f4-ec6c-474f-8db2-93de82248a28\": \"1e6cc8f4-ec6c-474f-8db2-93de82248a28\", \"d9441ba7-09ec-4726-ae88-df1863261a9f\": \"d9441ba7-09ec-4726-ae88-df1863261a9f\", \"a5aa5af4-12d1-4a01-8e0e-5318ca3b6de4\": \"a5aa5af4-12d1-4a01-8e0e-5318ca3b6de4\", \"009e44e3-4730-485f-add7-e1e517ee1536\": \"009e44e3-4730-485f-add7-e1e517ee1536\", \"3b0649c0-a990-4340-8d4d-b65db3cb432b\": \"3b0649c0-a990-4340-8d4d-b65db3cb432b\", \"bd7b998c-ead4-4d14-a01d-9c4c3d1d5d65\": \"bd7b998c-ead4-4d14-a01d-9c4c3d1d5d65\"}, \"doc_id_dict\": {}, \"embeddings_dict\": {}}"}}}
+{"index_store/data": {"da495c94-4541-47e1-b93f-8535192a5f28": {"__type__": "vector_store", "__data__": "{\"index_id\": \"da495c94-4541-47e1-b93f-8535192a5f28\", \"summary\": null, \"nodes_dict\": {\"59108663-a5e1-4e3e-bb21-626158eef136\": \"59108663-a5e1-4e3e-bb21-626158eef136\", \"50de4ec9-febb-466f-9f9a-cc9296895e83\": \"50de4ec9-febb-466f-9f9a-cc9296895e83\", \"aa413a53-0dda-4ac4-8ae9-6e8e340bb4f0\": \"aa413a53-0dda-4ac4-8ae9-6e8e340bb4f0\", \"a0cc4323-ec8f-4fed-9401-e44125134341\": \"a0cc4323-ec8f-4fed-9401-e44125134341\", \"5321cc7b-2a86-48b8-b56c-415dde7c149b\": \"5321cc7b-2a86-48b8-b56c-415dde7c149b\", \"9e19fb91-8258-4aca-9692-2d027073499e\": \"9e19fb91-8258-4aca-9692-2d027073499e\", \"02e856e5-4211-4a27-9204-e966907f1d74\": \"02e856e5-4211-4a27-9204-e966907f1d74\", \"f3074870-8fbf-4322-b1d2-2111e6aac9af\": \"f3074870-8fbf-4322-b1d2-2111e6aac9af\", \"82677fb9-abe3-4038-8263-5576c47da4f2\": \"82677fb9-abe3-4038-8263-5576c47da4f2\", \"a08364a6-c23d-4df5-8b5d-84137fbebd4e\": \"a08364a6-c23d-4df5-8b5d-84137fbebd4e\", \"e45b082d-c3ec-45aa-b630-6db49a62728b\": \"e45b082d-c3ec-45aa-b630-6db49a62728b\", \"2c55445c-04b1-4705-9871-adaa02f38f1b\": \"2c55445c-04b1-4705-9871-adaa02f38f1b\", \"d0de9736-ccad-450e-b4a1-49d4cdb8b941\": \"d0de9736-ccad-450e-b4a1-49d4cdb8b941\", \"fd0d2375-39e2-4bce-8e39-1182a122a1b4\": \"fd0d2375-39e2-4bce-8e39-1182a122a1b4\", \"13221de7-6c68-4367-b1be-f35b06fc3a74\": \"13221de7-6c68-4367-b1be-f35b06fc3a74\", \"9f448401-cda9-4b5f-9a80-c79e111f9963\": \"9f448401-cda9-4b5f-9a80-c79e111f9963\", \"3bc7dfc2-3ddf-4384-a60c-6cd52e1314f4\": \"3bc7dfc2-3ddf-4384-a60c-6cd52e1314f4\", \"ce3e530c-ce2d-4f5f-a171-72a790c3c624\": \"ce3e530c-ce2d-4f5f-a171-72a790c3c624\", \"85f764bd-e560-48ba-a51e-2287b6fe19db\": \"85f764bd-e560-48ba-a51e-2287b6fe19db\", \"3a8e4c7c-9f7d-4735-93e7-9d847cff98de\": \"3a8e4c7c-9f7d-4735-93e7-9d847cff98de\", \"af881b61-03f4-4851-8946-794015e3436c\": \"af881b61-03f4-4851-8946-794015e3436c\", \"31579820-439e-4029-b8c4-a0d6528daa59\": \"31579820-439e-4029-b8c4-a0d6528daa59\"}, \"doc_id_dict\": {}, \"embeddings_dict\": {}}"}}}

langchain_manager/manager.py

@@ -28,6 +28,11 @@ class LangChainAzureManager(BaseLangChainManager):
             # model_name="text-davinci-003",
             model="text-davinci-003",
             client=None,
+            # temperature set to 0.0(default 0.7) to get a certain answer from OpenAI,
+            # as a wiki robot we won't want to get flexible answers
+            temperature=0.0,
+            # GPT-3 default is 4096, however, openai.py default is 256
+            max_tokens=2048,
         )
 
     # Override

llama/service_context.py

@@ -1,13 +1,26 @@
 from abc import abstractmethod, ABC
 
-from llama_index import ServiceContext, LLMPredictor, LangchainEmbedding, Document
-from llama_index import StorageContext
-from typing import List
+from llama_index import ServiceContext, LLMPredictor, LangchainEmbedding
 
 from core.lifecycle import Lifecycle
 from langchain_manager.manager import BaseLangChainManager
 
 
+# def get_callback_manager() -> CallbackManager:
+#     from llama_index.callbacks import (
+#         WandbCallbackHandler,
+#         CallbackManager,
+#         LlamaDebugHandler,
+#     )
+#     llama_debug = LlamaDebugHandler(print_trace_on_end=True)
+#     # wandb.init args
+#     run_args = dict(
+#         project="llamaindex",
+#     )
+#     wandb_callback = WandbCallbackHandler(run_args=run_args)
+#     return CallbackManager([llama_debug, wandb_callback])
+
+
 class ServiceContextManager(Lifecycle, ABC):
     @abstractmethod
     def get_service_context(self) -> ServiceContext:
@@ -36,7 +49,9 @@ class AzureServiceContextManager(ServiceContextManager):
         llm_predictor = LLMPredictor(llm=self.lc_manager.get_llm())
         # configure service context
         self.service_context = ServiceContext.from_defaults(
-            llm_predictor=llm_predictor, embed_model=embedding
+            llm_predictor=llm_predictor,
+            embed_model=embedding,
+            # callback_manager=get_callback_manager(),
         )
 
     def do_start(self) -> None:
@@ -95,7 +110,9 @@ class HuggingFaceChineseOptServiceContextManager(ServiceContextManager):
         llm_predictor = LLMPredictor(self.lc_manager.get_llm())
         # configure service context
         self.service_context = ServiceContext.from_defaults(
-            llm_predictor=llm_predictor, embed_model=embedding
+            llm_predictor=llm_predictor,
+            embed_model=embedding,
+            # callback_manager=get_callback_manager()
         )
 
     def do_start(self) -> None:
@@ -123,62 +140,3 @@ class HuggingFaceChineseOptServiceContextManager(ServiceContextManager):
             "[do_dispose] total used token: %d",
             self.service_context.llm_predictor.total_tokens_used,
         )
-
-
-class StorageContextManager(Lifecycle, ABC):
-    @abstractmethod
-    def get_storage_context(self) -> StorageContext:
-        pass
-
-
-class LocalStorageContextManager(StorageContextManager):
-    storage_context: StorageContext
-
-    def __init__(
-        self,
-        service_context_manager: ServiceContextManager,
-        dataset_path: str = "./dataset",
-    ) -> None:
-        super().__init__()
-        self.dataset_path = dataset_path
-        self.service_context_manager = service_context_manager
-
-    def get_storage_context(self) -> StorageContext:
-        return self.storage_context
-
-    def do_init(self) -> None:
-        from llama.utils import is_local_storage_files_ready
-
-        if is_local_storage_files_ready(self.dataset_path):
-            self.storage_context = StorageContext.from_defaults(
-                persist_dir=self.dataset_path
-            )
-        else:
-            docs = self._download()
-            self._indexing(docs)
-
-    def do_start(self) -> None:
-        # self.logger.info("[do_start]%", **self.storage_context.to_dict())
-        pass
-
-    def do_stop(self) -> None:
-        # self.logger.info("[do_stop]%", **self.storage_context.to_dict())
-        pass
-
-    def do_dispose(self) -> None:
-        self.storage_context.persist(self.dataset_path)
-
-    def _download(self) -> List[Document]:
-        from llama.data_loader import GithubLoader
-
-        loader = GithubLoader()
-        return loader.load()
-
-    def _indexing(self, docs: List[Document]) -> None:
-        from llama_index import GPTVectorStoreIndex
-
-        index = GPTVectorStoreIndex.from_documents(
-            docs, service_context=self.service_context_manager.get_service_context()
-        )
-        index.storage_context.persist(persist_dir=self.dataset_path)
-        self.storage_context = index.storage_context

llama/storage_context.py

@@ -0,0 +1,67 @@
+from llama_index import StorageContext
+from typing import List
+from abc import abstractmethod, ABC
+
+from llama_index import Document
+
+from core.lifecycle import Lifecycle
+from llama.service_context import ServiceContextManager
+
+
+class StorageContextManager(Lifecycle, ABC):
+    @abstractmethod
+    def get_storage_context(self) -> StorageContext:
+        pass
+
+
+class LocalStorageContextManager(StorageContextManager):
+    storage_context: StorageContext
+
+    def __init__(
+        self,
+        service_context_manager: ServiceContextManager,
+        dataset_path: str = "./dataset",
+    ) -> None:
+        super().__init__()
+        self.dataset_path = dataset_path
+        self.service_context_manager = service_context_manager
+
+    def get_storage_context(self) -> StorageContext:
+        return self.storage_context
+
+    def do_init(self) -> None:
+        from llama.utils import is_local_storage_files_ready
+
+        if is_local_storage_files_ready(self.dataset_path):
+            self.storage_context = StorageContext.from_defaults(
+                persist_dir=self.dataset_path
+            )
+        else:
+            docs = self._download()
+            self._indexing(docs)
+
+    def do_start(self) -> None:
+        # self.logger.info("[do_start]%", **self.storage_context.to_dict())
+        pass
+
+    def do_stop(self) -> None:
+        # self.logger.info("[do_stop]%", **self.storage_context.to_dict())
+        pass
+
+    def do_dispose(self) -> None:
+        self.storage_context.persist(self.dataset_path)
+
+    def _download(self) -> List[Document]:
+        from llama.data_loader import GithubLoader
+
+        loader = GithubLoader()
+        return loader.load()
+
+    def _indexing(self, docs: List[Document]) -> None:
+        from llama_index import GPTVectorStoreIndex
+
+        index = GPTVectorStoreIndex.from_documents(
+            docs, service_context=self.service_context_manager.get_service_context()
+        )
+        index.storage_context.persist(persist_dir=self.dataset_path)
+        self.storage_context = index.storage_context

requirements.txt

@@ -6,4 +6,5 @@ black
 mypy
 accelerate
 python-dotenv
-sentence_transformers
+sentence_transformers
+wandb

xpipe_wiki/manager_factory.py

@@ -42,7 +42,7 @@ class XPipeRobotManagerFactory:
         service_context_manager = AzureServiceContextManager(
             lc_manager=LangChainAzureManager()
         )
-        from llama.service_context import LocalStorageContextManager
+        from llama.storage_context import LocalStorageContextManager
 
         dataset_path = os.getenv("XPIPE_WIKI_DATASET_PATH", "./dataset")
         storage_context_manager = LocalStorageContextManager(
@@ -66,7 +66,7 @@ class XPipeRobotManagerFactory:
             lc_manager=LangChainAzureManager()
         )
 
-        from llama.service_context import LocalStorageContextManager
+        from llama.storage_context import LocalStorageContextManager
 
         dataset_path = os.getenv("XPIPE_WIKI_DATASET_PATH", "./dataset")
         storage_context_manager = LocalStorageContextManager(

xpipe_wiki/robot_manager.py

@@ -3,10 +3,12 @@ from typing import Any
 
 from llama_index import load_index_from_storage
 from llama_index.indices.query.base import BaseQueryEngine
+from llama_index.indices.response import ResponseMode
 
 from core.helper import LifecycleHelper
 from core.lifecycle import Lifecycle
-from llama.service_context import ServiceContextManager, StorageContextManager
+from llama.service_context import ServiceContextManager
+from llama.storage_context import StorageContextManager
 
 
 class XPipeWikiRobot(ABC):
@@ -23,7 +25,10 @@ class AzureOpenAIXPipeWikiRobot(XPipeWikiRobot):
         self.query_engine = query_engine
 
     def ask(self, question: str) -> Any:
-        return self.query_engine.query(question).response
+        print("question: ", question)
+        response = self.query_engine.query(question)
+        print("response type: ", type(response))
+        return response.__str__()
 
 
 class XPipeWikiRobotManager(Lifecycle):
@@ -61,7 +66,8 @@ class AzureXPipeWikiRobotManager(XPipeWikiRobotManager):
             service_context=self.service_context_manager.get_service_context(),
         )
         self.query_engine = index.as_query_engine(
-            service_context=self.service_context_manager.get_service_context()
+            service_context=self.service_context_manager.get_service_context(),
+            response_mode=ResponseMode.REFINE,
         )
 
     def do_stop(self) -> None: