init hugging

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 TrialAndErrorAI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

app.py

@@ -0,0 +1,48 @@
+import streamlit as st
+from streamlit_chat import message
+from memory import get_memory
+
+print('this is app.py')
+# st.title('How My ZSH')
+
+st.markdown(""" 
+    # How My ZSH
+    This is a demo of a chatbot that can answer questions about the Oh My Zsh
+    wiki. It uses the [Streamlit Chat]() component and the [Chroma]() library to 
+    search the wiki and find answers to your questions. 
+""")
+            
+def get_text():
+    input_text = st.text_input("", key="input")
+    return input_text
+
+# Initialize the session state for generated responses and past inputs
+if 'generated' not in st.session_state:
+    st.session_state['generated'] = ['i am ready to help you ser']
+
+if 'past' not in st.session_state:
+    st.session_state['past'] = ['hello']
+
+# Get the user's input from the text input field
+user_input = get_text()
+
+# add reset or clear button to clear the chat
+if st.button('Clear', key='clear'):
+    st.session_state['generated'] = []
+    st.session_state['past'] = []
+
+# If there is user input, search for a response using the search_chroma function
+if user_input:
+    with st.spinner(text='Searching for response...'):
+        output = get_memory().query(user_input)
+    st.session_state.past.append(user_input)
+    st.session_state.generated.append(output)
+
+# If there are generated responses, display the conversation using Streamlit messages
+if st.session_state['generated']:
+    for i in range(len(st.session_state['generated'])):
+        message(st.session_state['past'][i],
+                avatar_style="personas",
+                is_user=True, key=str(i) + '_user')
+        message(st.session_state["generated"][i], key=str(i))
+        

memory.py

@@ -0,0 +1,88 @@
+import os
+from dotenv import load_dotenv
+from langchain import OpenAI
+from langchain.document_loaders import GitLoader
+from langchain.vectorstores import VectorStore, Chroma
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.chains.question_answering import load_qa_chain
+
+REPO_PATH = "out/ohmyzsh-wiki"
+OHMYZSH_WIKI_URL = "https://github.com/ohmyzsh/wiki"
+PRESISTENT_PATH = ".vdb"
+DEBUG = True
+
+memory = None
+
+print('this is memory.py')
+load_dotenv()
+
+class Memory: 
+    _instance = None
+    vectordb = None
+
+    def __init__(self):
+        if self._instance is not None:
+            raise Exception("Memory is a singleton")
+        else:
+            Memory._instance = self
+            self.vectordb = self.update_memory()
+    
+    @staticmethod
+    def get_instance():
+        if Memory._instance is None:
+            Memory()
+        return Memory._instance
+    
+
+
+    # TODO - make it singleton and load it once
+    def query(self, query):
+        if self.vectordb is None:
+            self.vectordb = Chroma(persist_directory=PRESISTENT_PATH, embedding_function=OpenAIEmbeddings())
+
+        context_docs = self.vectordb.as_retriever().get_relevant_documents(query)
+        chain = load_qa_chain(llm=OpenAI(), chain_type="stuff")
+        answer = chain.run(input_documents=context_docs, question=query)
+
+        if DEBUG:
+            print(f"Found {len(context_docs)} relevant documents")
+            print(f"Answer: {answer}")
+
+        return answer
+    
+    ## Private methods
+    def update_memory(self):
+        docs = self._sniff_github(clone_url=None if os.path.exists(REPO_PATH) else OHMYZSH_WIKI_URL, branch="main", file_ext="md")
+        text_splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=100)
+        doc_chunks = text_splitter.split_documents(docs)
+        vectordb = Chroma.from_documents(documents=doc_chunks, embedding=OpenAIEmbeddings(), persist_directory=PRESISTENT_PATH)
+        vectordb.persist()
+        if DEBUG:
+            print(f"[[Memory Update]]. Added {len(doc_chunks)} vectors to the vector store.")
+        return vectordb
+    
+    def _sniff_github(self, clone_url=None, branch="main", file_ext=None):
+        loader = GitLoader(repo_path="out/ohmyzsh-wiki",
+                        clone_url=clone_url,
+                        branch=branch, 
+                        file_filter= lambda file_path: file_path.endswith(f".{file_ext}")) if file_ext else None
+
+        documents = loader.load()
+        if DEBUG:
+            print(f"Found {len(documents)} documents in repo\n")    
+        return documents
+
+def get_memory():
+    global memory
+    if memory is None:
+        return Memory.get_instance()
+    else:
+        return memory
+
+def main():
+    memory = Memory.get_instance()
+    print(memory.query("how to install ohmyzsh"))
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,105 @@
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==4.2.2
+anyio==3.6.2
+async-timeout==4.0.2
+attrs==23.1.0
+backoff==2.2.1
+blinker==1.6.2
+cachetools==5.3.0
+certifi==2022.12.7
+charset-normalizer==3.1.0
+chromadb==0.3.21
+click==8.1.3
+clickhouse-connect==0.5.21
+dataclasses-json==0.5.7
+decorator==5.1.1
+duckdb==0.7.1
+entrypoints==0.4
+fastapi==0.95.1
+filelock==3.12.0
+frozenlist==1.3.3
+fsspec==2023.4.0
+gitdb==4.0.10
+GitPython==3.1.31
+h11==0.14.0
+hnswlib==0.7.0
+httptools==0.5.0
+huggingface-hub==0.14.1
+idna==3.4
+importlib-metadata==6.6.0
+Jinja2==3.1.2
+joblib==1.2.0
+jsonschema==4.17.3
+langchain==0.0.150
+lz4==4.3.2
+markdown-it-py==2.2.0
+MarkupSafe==2.1.2
+marshmallow==3.19.0
+marshmallow-enum==1.5.1
+mdurl==0.1.2
+monotonic==1.6
+mpmath==1.3.0
+multidict==6.0.4
+mypy-extensions==1.0.0
+networkx==3.1
+nltk==3.8.1
+numexpr==2.8.4
+numpy==1.24.3
+openai==0.27.5
+openapi-schema-pydantic==1.2.4
+packaging==23.1
+pandas==2.0.1
+Pillow==9.5.0
+posthog==3.0.1
+protobuf==3.20.3
+pyarrow==11.0.0
+pydantic==1.10.7
+pydeck==0.8.1b0
+Pygments==2.15.1
+Pympler==1.0.1
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+pytz==2023.3
+pytz-deprecation-shim==0.1.0.post0
+PyYAML==6.0
+regex==2023.3.23
+requests==2.29.0
+rich==13.3.5
+scikit-learn==1.2.2
+scipy==1.10.1
+sentence-transformers==2.2.2
+sentencepiece==0.1.98
+six==1.16.0
+smmap==5.0.0
+sniffio==1.3.0
+SQLAlchemy==2.0.11
+starlette==0.26.1
+streamlit==1.22.0
+streamlit-chat==0.0.2.2
+sympy==1.11.1
+tenacity==8.2.2
+threadpoolctl==3.1.0
+tiktoken==0.3.3
+tokenizers==0.13.3
+toml==0.10.2
+toolz==0.12.0
+torch==2.0.0
+torchvision==0.15.1
+tornado==6.3.1
+tqdm==4.65.0
+transformers==4.28.1
+typing-inspect==0.8.0
+typing_extensions==4.5.0
+tzdata==2023.3
+tzlocal==4.3
+urllib3==1.26.15
+uvicorn==0.21.1
+uvloop==0.17.0
+validators==0.20.0
+watchfiles==0.19.0
+websockets==11.0.2
+yarl==1.9.2
+zipp==3.15.0
+zstandard==0.21.0