Merge pull request #98 from edubotics-ai/dev_branch

Dev branch to Main

Author: trgardos

.github/workflows/publish.yml

@@ -0,0 +1,29 @@
+name: Publish to PyPI
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - "v*"
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel twine
+      - name: Build and publish
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} # from pypi
+        run: |
+          python setup.py sdist bdist_wheel
+          twine upload dist/*

.github/workflows/push_to_hf_space.yml

@@ -18,4 +18,4 @@ jobs:
       - name: Deploy Production (main) to HuggingFace
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
-        run: git push --force https://trgardos:[email protected]/spaces/dl4ds/dl4ds_tutor main:main
+        run: git push --force https://trgardos:[email protected]/spaces/edubotics/dl4ds_tutor main:main

.vscode/launch.json

@@ -17,7 +17,7 @@
         {   "name":"Python Debugger: Module store_manager",
             "type":"debugpy",
             "request":"launch",
-            "module":"modules.vectorstore.store_manager",
+            "module":"edubotics_core.vectorstore.store_manager",
             "env": {"PYTHONPATH": "${workspaceFolder}/code"},
             "cwd": "${workspaceFolder}/code",
             "justMyCode": true
@@ -26,7 +26,7 @@
             "name": "Python Debugger: Module data_loader",
             "type": "debugpy",
             "request": "launch",
-            "module": "modules.dataloader.data_loader",
+            "module": "edubotics_core.dataloader.data_loader",
             "env": {"PYTHONPATH": "${workspaceFolder}/code"},
             "cwd": "${workspaceFolder}/code",
             "justMyCode": true

Dockerfile

@@ -40,4 +40,4 @@ RUN --mount=type=secret,id=LITERAL_API_KEY_LOGGING,mode=0444,required=true
 RUN --mount=type=secret,id=CHAINLIT_AUTH_SECRET,mode=0444,required=true 
 
 # Default command to run the application
-CMD python -m modules.vectorstore.store_manager --config_file config/config.yml --project_config_file config/project_config.yml && python -m uvicorn app:app --host 0.0.0.0 --port 7860
+CMD python -m edubotics_core.vectorstore.store_manager --config_file config/config.yml --project_config_file config/project_config.yml && python -m uvicorn app:app --host 0.0.0.0 --port 7860

Dockerfile.dev

@@ -35,4 +35,4 @@ RUN ls -R /code
 EXPOSE 7860
 
 # Default command to run the application
-CMD python -m modules.vectorstore.store_manager --config_file config/config.yml --project_config_file config/project_config.yml && python -m uvicorn app:app --host 0.0.0.0 --port 7860
+CMD python -m edubotics_core.vectorstore.store_manager --config_file config/config.yml --project_config_file config/project_config.yml && python -m uvicorn app:app --host 0.0.0.0 --port 7860

README.md

@@ -65,7 +65,7 @@ default_collapse_content = true
 cot = "hidden"
 
 # Link to your github repo. This will add a github button in the UI's header.
-github = "https://github.com/DL4DS/dl4ds_tutor"
+github = "https://github.com/edubotics-ai/edubot-core"
 
 # Specify a CSS file that can be used to customize the user interface.
 # The CSS file can be served from the public directory or via an external link.

apps/ai_tutor/.chainlit/config.toml

@@ -21,7 +21,7 @@
     reset_tokens_for_user,
     check_user_cooldown,
 )
-from modules.chat_processor.helpers import get_user_details, update_user_info
+from edubotics_core.chat_processor.helpers import get_user_details, update_user_info
 from config.config_manager import config_manager
 import hashlib
 

apps/ai_tutor/app.py

@@ -4,18 +4,18 @@
     LITERAL_API_KEY_LOGGING,
     LITERAL_API_URL,
 )
-from modules.chat_processor.literal_ai import CustomLiteralDataLayer
+from edubotics_core.chat_processor.literal_ai import CustomLiteralDataLayer
 import json
 from typing import Any, Dict, no_type_check
 import chainlit as cl
-from modules.chat.llm_tutor import LLMTutor
-from modules.chat.helpers import (
+from edubotics_core.chat.llm_tutor import LLMTutor
+from edubotics_core.chat.helpers import (
     get_sources,
     get_history_chat_resume,
     get_history_setup_llm,
     # get_last_config,
 )
-from modules.chat_processor.helpers import (
+from edubotics_core.chat_processor.helpers import (
     update_user_info,
     get_user_details,
 )

apps/ai_tutor/chainlit_app.py

@@ -104,7 +104,7 @@ class TokenConfig(BaseModel):
 
 
 class MiscConfig(BaseModel):
-    github_repo: HttpUrl = "https://github.com/DL4DS/dl4ds_tutor"
+    github_repo: HttpUrl = "https://github.com/edubotics-ai/edubot-core"
     docs_website: HttpUrl = "https://dl4ds.github.io/dl4ds_tutor/"
 
 

apps/ai_tutor/config/config_manager.py

@@ -13,7 +13,7 @@ token_config:
   all_time_tokens_allocated: 1000000
 
 misc:
-  github_repo: "https://github.com/DL4DS/dl4ds_tutor"
+  github_repo: "https://github.com/edubotics-ai/edubot-core"
   docs_website: "https://dl4ds.github.io/dl4ds_tutor/"
 
 api_config:

apps/ai_tutor/config/project_config.yml

@@ -1,6 +1,6 @@
 from datetime import datetime, timedelta, timezone
 import tiktoken
-from modules.chat_processor.helpers import update_user_info, convert_to_dict
+from edubotics_core.chat_processor.helpers import update_user_info, convert_to_dict
 
 
 def get_time():

apps/ai_tutor/helpers.py

@@ -2,8 +2,8 @@
 import asyncio
 from typing import Any, Dict, no_type_check
 import chainlit as cl
-from modules.chat.llm_tutor import LLMTutor
-from modules.chat.helpers import (
+from edubotics_core.chat.llm_tutor import LLMTutor
+from edubotics_core.chat.helpers import (
     get_sources,
     get_history_setup_llm,
 )

apps/chainlit_base/chainlit_base.py

@@ -1,8 +1,8 @@
 from langchain_core.prompts import ChatPromptTemplate
 
-# from modules.chat.langchain.utils import
+# from edubotics_core.chat.langchain.utils import
 from langchain_community.chat_message_histories import ChatMessageHistory
-from modules.chat.base import BaseRAG
+from edubotics_core.chat.base import BaseRAG
 from langchain_core.prompts import PromptTemplate
 from langchain.memory import ConversationBufferWindowMemory
 from langchain_core.runnables.utils import ConfigurableFieldSpec

modules/__init__.py renamed to edubotics_core/__init__.py

@@ -1,8 +1,8 @@
-from modules.chat.helpers import get_prompt
-from modules.chat.chat_model_loader import ChatModelLoader
-from modules.vectorstore.store_manager import VectorStoreManager
-from modules.retriever.retriever import Retriever
-from modules.chat.langchain.langchain_rag import (
+from edubotics_core.chat.helpers import get_prompt
+from edubotics_core.chat.chat_model_loader import ChatModelLoader
+from edubotics_core.vectorstore.store_manager import VectorStoreManager
+from edubotics_core.retriever.retriever import Retriever
+from edubotics_core.chat.langchain.langchain_rag import (
     Langchain_RAG_V2,
     QuestionGenerator,
 )

modules/chat/__init__.py renamed to edubotics_core/chat/__init__.py

@@ -18,11 +18,11 @@
 import html2text
 import bs4
 import PyPDF2
-from modules.dataloader.pdf_readers.base import PDFReader
-from modules.dataloader.pdf_readers.llama import LlamaParser
-from modules.dataloader.pdf_readers.gpt import GPTParser
-from modules.dataloader.helpers import get_metadata
-from modules.config.constants import TIMEOUT
+from edubotics_core.dataloader.pdf_readers.base import PDFReader
+from edubotics_core.dataloader.pdf_readers.llama import LlamaParser
+from edubotics_core.dataloader.pdf_readers.gpt import GPTParser
+from edubotics_core.dataloader.helpers import get_metadata
+from edubotics_core.config.constants import TIMEOUT
 
 logger = logging.getLogger(__name__)
 BASE_DIR = os.getcwd()

modules/chat/base.py renamed to edubotics_core/chat/base.py

@@ -2,7 +2,7 @@
 from bs4 import BeautifulSoup
 from urllib.parse import urlparse
 import tempfile
-from modules.config.constants import (
+from edubotics_core.config.constants import (
     TIMEOUT,
 )  # TODO: MOVE THIS TO APP SPECIFIC DIRECTORY
 

modules/chat/chat_model_loader.py renamed to edubotics_core/chat/chat_model_loader.py

@@ -6,7 +6,7 @@
 from openai import OpenAI
 from pdf2image import convert_from_path
 from langchain.schema import Document
-from modules.config.constants import TIMEOUT
+from edubotics_core.config.constants import TIMEOUT
 
 
 class GPTParser:

modules/chat/helpers.py renamed to edubotics_core/chat/helpers.py

@@ -2,8 +2,8 @@
 import requests
 from llama_parse import LlamaParse
 from langchain.schema import Document
-from modules.config.constants import OPENAI_API_KEY, LLAMA_CLOUD_API_KEY, TIMEOUT
-from modules.dataloader.helpers import download_pdf_from_url
+from edubotics_core.config.constants import OPENAI_API_KEY, LLAMA_CLOUD_API_KEY, TIMEOUT
+from edubotics_core.dataloader.helpers import download_pdf_from_url
 
 
 class LlamaParser:

modules/chat/langchain/__init__.py renamed to edubotics_core/chat/langchain/__init__.py

@@ -4,7 +4,7 @@
 import requests
 from bs4 import BeautifulSoup
 from urllib.parse import urljoin, urldefrag
-from modules.config.constants import TIMEOUT
+from edubotics_core.config.constants import TIMEOUT
 
 
 class WebpageCrawler:

modules/chat/langchain/langchain_rag.py renamed to edubotics_core/chat/langchain/langchain_rag.py

@@ -1,7 +1,7 @@
-from modules.retriever.faiss_retriever import FaissRetriever
-from modules.retriever.chroma_retriever import ChromaRetriever
-from modules.retriever.colbert_retriever import ColbertRetriever
-from modules.retriever.raptor_retriever import RaptorRetriever
+from edubotics_core.retriever.faiss_retriever import FaissRetriever
+from edubotics_core.retriever.chroma_retriever import ChromaRetriever
+from edubotics_core.retriever.colbert_retriever import ColbertRetriever
+from edubotics_core.retriever.raptor_retriever import RaptorRetriever
 
 
 class Retriever:

modules/chat/langchain/utils.py renamed to edubotics_core/chat/langchain/utils.py

@@ -1,5 +1,5 @@
 from langchain_community.vectorstores import Chroma
-from modules.vectorstore.base import VectorStoreBase
+from edubotics_core.vectorstore.base import VectorStoreBase
 import os
 
 

modules/chat/llm_tutor.py renamed to edubotics_core/chat/llm_tutor.py

@@ -1,5 +1,5 @@
 from ragatouille import RAGPretrainedModel
-from modules.vectorstore.base import VectorStoreBase
+from edubotics_core.vectorstore.base import VectorStoreBase
 from langchain_core.retrievers import BaseRetriever
 from langchain_core.callbacks.manager import CallbackManagerForRetrieverRun
 from langchain_core.documents import Document

modules/chat_processor/__init__.py renamed to edubotics_core/chat_processor/__init__.py

@@ -1,6 +1,6 @@
 from langchain_community.embeddings import OpenAIEmbeddings
 from langchain_community.embeddings import HuggingFaceEmbeddings
-from modules.config.constants import OPENAI_API_KEY, HUGGINGFACE_TOKEN
+from edubotics_core.config.constants import OPENAI_API_KEY, HUGGINGFACE_TOKEN
 
 
 class EmbeddingModelLoader:

modules/chat_processor/helpers.py renamed to edubotics_core/chat_processor/helpers.py

@@ -1,5 +1,5 @@
 from langchain_community.vectorstores import FAISS
-from modules.vectorstore.base import VectorStoreBase
+from edubotics_core.vectorstore.base import VectorStoreBase
 import os
 
 

modules/chat_processor/literal_ai.py renamed to edubotics_core/chat_processor/literal_ai.py

@@ -11,7 +11,7 @@
 from langchain_community.chat_models import ChatOpenAI
 from langchain_community.vectorstores import FAISS
 from langchain.text_splitter import RecursiveCharacterTextSplitter
-from modules.vectorstore.base import VectorStoreBase
+from edubotics_core.vectorstore.base import VectorStoreBase
 
 RANDOM_SEED = 42
 

modules/config/__init__.py renamed to edubotics_core/config/__init__.py

@@ -1,12 +1,14 @@
-from modules.vectorstore.vectorstore import VectorStore
-from modules.dataloader.helpers import get_urls_from_file
-from modules.dataloader.webpage_crawler import WebpageCrawler
-from modules.dataloader.data_loader import DataLoader
-from modules.vectorstore.embedding_model_loader import EmbeddingModelLoader
+from edubotics_core.vectorstore.vectorstore import VectorStore
+from edubotics_core.dataloader.helpers import get_urls_from_file
+from edubotics_core.dataloader.webpage_crawler import WebpageCrawler
+from edubotics_core.dataloader.data_loader import DataLoader
+from edubotics_core.vectorstore.embedding_model_loader import EmbeddingModelLoader
 import logging
 import os
 import time
 import asyncio
+import yaml
+import argparse
 
 
 class VectorStoreManager:
@@ -137,7 +139,7 @@ def load_database(self):
             self.loaded_vector_db = self.vector_db._load_database(self.embedding_model)
         except Exception as e:
             raise ValueError(
-                f"Error loading database, check if it exists. if not run python -m modules.vectorstore.store_manager / Resteart the HF Space: {e}"
+                f"Error loading database, check if it exists. if not run python -m edubotics_core.vectorstore.store_manager / Resteart the HF Space: {e}"
             )
             # print(f"Creating database")
             # self.create_database()
@@ -162,10 +164,7 @@ def __len__(self):
         return len(self.vector_db)
 
 
-if __name__ == "__main__":
-    import yaml
-    import argparse
-
+def main():
     # Add argument parsing for config files
     parser = argparse.ArgumentParser(description="Load configuration files.")
     parser.add_argument(
@@ -216,3 +215,7 @@ def __len__(self):
     print("Loaded database")
 
     print(f"View the logs at {config['log_dir']}/vector_db.log")
+
+
+if __name__ == "__main__":
+    main()

modules/config/constants.py renamed to edubotics_core/config/constants.py

@@ -1,7 +1,7 @@
-from modules.vectorstore.faiss import FaissVectorStore
-from modules.vectorstore.chroma import ChromaVectorStore
-from modules.vectorstore.colbert import ColbertVectorStore
-from modules.vectorstore.raptor import RAPTORVectoreStore
+from edubotics_core.vectorstore.faiss import FaissVectorStore
+from edubotics_core.vectorstore.chroma import ChromaVectorStore
+from edubotics_core.vectorstore.colbert import ColbertVectorStore
+from edubotics_core.vectorstore.raptor import RAPTORVectoreStore
 from huggingface_hub import snapshot_download
 import os
 import shutil