Merge pull request #90 from DL4DS/dev2main

Update main with current state of dev

Author: trgardos

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+max-line-length = 88
+extend-ignore = E203, E266, E501, W503

.gitattributes

@@ -0,0 +1 @@
+*.jpg filter=lfs diff=lfs merge=lfs -text

.github/workflows/code_quality_check.yml

@@ -0,0 +1,33 @@
+name: Code Quality and Security Checks
+
+on:
+  push:
+    branches: [ main, dev_branch ]
+  pull_request:
+    branches: [ main, dev_branch ]
+
+jobs:
+  code-quality:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+    
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install flake8 black bandit
+    
+    - name: Run Black
+      run: black --check .
+    
+    - name: Run Flake8
+      run: flake8 .
+    
+    - name: Run Bandit
+      run: |
+        bandit -r .

.gitignore

@@ -165,7 +165,9 @@ cython_debug/
 .ragatouille/*
 */__pycache__/*
 .chainlit/translations/
+code/.chainlit/translations/
 storage/logs/*
 vectorstores/*
 
-*/.files/*
+*/.files/*
+code/storage/models/

Dockerfile

@@ -26,6 +26,13 @@ WORKDIR /code/code
 
 RUN --mount=type=secret,id=HUGGINGFACEHUB_API_TOKEN,mode=0444,required=true 
 RUN --mount=type=secret,id=OPENAI_API_KEY,mode=0444,required=true 
+RUN --mount=type=secret,id=CHAINLIT_URL,mode=0444,required=true 
+RUN --mount=type=secret,id=LITERAL_API_URL,mode=0444,required=true 
+RUN --mount=type=secret,id=LLAMA_CLOUD_API_KEY,mode=0444,required=true 
+RUN --mount=type=secret,id=OAUTH_GOOGLE_CLIENT_ID,mode=0444,required=true 
+RUN --mount=type=secret,id=OAUTH_GOOGLE_CLIENT_SECRET,mode=0444,required=true 
+RUN --mount=type=secret,id=LITERAL_API_KEY_LOGGING,mode=0444,required=true 
+RUN --mount=type=secret,id=CHAINLIT_AUTH_SECRET,mode=0444,required=true 
 
 # Default command to run the application
-CMD ["sh", "-c", "python -m modules.vectorstore.store_manager && chainlit run main.py --host 0.0.0.0 --port 7860"]
+CMD ["sh", "-c", "python -m modules.vectorstore.store_manager && uvicorn app:app --host 0.0.0.0 --port 7860"]

README.md

@@ -15,10 +15,14 @@ You can find a "production" implementation of the Tutor running live at [DL4DS T
 Hugging Face [Space](https://huggingface.co/spaces/dl4ds/dl4ds_tutor). It is pushed automatically from the `main` branch of this repo by this
 [Actions Workflow](https://github.com/DL4DS/dl4ds_tutor/blob/main/.github/workflows/push_to_hf_space.yml) upon a push to `main`.
 
-A "development" version of the Tutor is running live at [DL4DS Tutor -- Dev](https://dl4ds-tutor-dev.hf.space) from this Hugging Face
+
+A "development" version of the Tutor is running live at [DL4DS Tutor -- Dev](https://dl4ds-tutor-dev.hf.space/) from this Hugging Face
 [Space](https://huggingface.co/spaces/dl4ds/tutor_dev). It is pushed automatically from the `dev_branch` branch of this repo by this
 [Actions Workflow](https://github.com/DL4DS/dl4ds_tutor/blob/dev_branch/.github/workflows/push_to_hf_space_prototype.yml) upon a push to `dev_branch`.
 
+## Setup
+
+Please visit [setup](https://dl4ds.github.io/dl4ds_tutor/guide/setup/) for more information on setting up the project.
 
 ## Running Locally
 
@@ -34,7 +38,7 @@ A "development" version of the Tutor is running live at [DL4DS Tutor -- Dev](htt
 3. **To test Data Loading (Optional)**
    ```bash
    cd code
-   python -m modules.dataloader.data_loader
+   python -m modules.dataloader.data_loader --links "your_pdf_link"
    ```
 
 4. **Create the Vector Database**
@@ -43,47 +47,16 @@ A "development" version of the Tutor is running live at [DL4DS Tutor -- Dev](htt
    python -m modules.vectorstore.store_manager
    ```
    - Note: You need to run the above command when you add new data to the `storage/data` directory, or if the `storage/data/urls.txt` file is updated.
-   - Alternatively, you can set `["vectorstore"]["embedd_files"]` to `True` in the `code/modules/config/config.yaml` file, which will embed files from the storage directory every time you run the below chainlit command.
 
-5. **Run the Chainlit App**
+6. **Run the FastAPI App**
    ```bash
-   chainlit run main.py
+   cd code
+   uvicorn app:app --port 7860 
    ```
 
-See the [docs](https://github.com/DL4DS/dl4ds_tutor/tree/main/docs) for more information.
-
-## File Structure
-
-```plaintext
-code/
- ├── modules
- │   ├── chat                # Contains the chatbot implementation
- │   ├── chat_processor      # Contains the implementation to process and log the conversations
- │   ├── config              # Contains the configuration files
- │   ├── dataloader          # Contains the implementation to load the data from the storage directory
- │   ├── retriever           # Contains the implementation to create the retriever
- │   └── vectorstore         # Contains the implementation to create the vector database
- ├── public
- │   ├── logo_dark.png       # Dark theme logo
- │   ├── logo_light.png      # Light theme logo
- │   └── test.css            # Custom CSS file
- └── main.py
-
- 
-docs/                        # Contains the documentation to the codebase and methods used
+## Documentation
 
-storage/
- ├── data                    # Store files and URLs here
- ├── logs                    # Logs directory, includes logs on vector DB creation, tutor logs, and chunks logged in JSON files
- └── models                  # Local LLMs are loaded from here
-
-vectorstores/                # Stores the created vector databases
-
-.env                         # This needs to be created, store the API keys here
-```
-- `code/modules/vectorstore/vectorstore.py`: Instantiates the `VectorStore` class to create the vector database.
-- `code/modules/vectorstore/store_manager.py`: Instantiates the `VectorStoreManager:` class to manage the vector database, and all associated methods.
-- `code/modules/retriever/retriever.py`: Instantiates the `Retriever` class to create the retriever.
+Please visit the [docs](https://dl4ds.github.io/dl4ds_tutor/) for more information.
 
 
 ## Docker 
@@ -97,4 +70,10 @@ docker run -it --rm -p 8000:8000 dev
 
 ## Contributing
 
-Please create an issue if you have any suggestions or improvements, and start working on it by creating a branch and by making a pull request to the main branch.
+Please create an issue if you have any suggestions or improvements, and start working on it by creating a branch and by making a pull request to the `dev_branch`.
+
+Please visit [contribute](https://dl4ds.github.io/dl4ds_tutor/guide/contribute/) for more information on contributing.
+
+## Future Work
+
+For more information on future work, please visit [roadmap](https://dl4ds.github.io/dl4ds_tutor/guide/readmap/).

code/.chainlit/config.toml

@@ -20,7 +20,7 @@ allow_origins = ["*"]
 
 [features]
 # Process and display HTML in messages. This can be a security risk (see https://stackoverflow.com/questions/19603097/why-is-it-dangerous-to-render-user-generated-html-or-javascript)
-unsafe_allow_html = false
+unsafe_allow_html = true
 
 # Process and display mathematical expressions. This can clash with "$" characters in messages.
 latex = true
@@ -49,6 +49,8 @@ auto_tag_thread = true
     # Sample rate of the audio
     sample_rate = 44100
 
+edit_message = true
+
 [UI]
 # Name of the assistant.
 name = "AI Tutor"
@@ -59,11 +61,11 @@ name = "AI Tutor"
 # Large size content are by default collapsed for a cleaner ui
 default_collapse_content = true
 
-# Hide the chain of thought details from the user in the UI.
-hide_cot = true
+# Chain of Thought (CoT) display mode. Can be "hidden", "tool_call" or "full".
+cot = "hidden"
 
 # Link to your github repo. This will add a github button in the UI's header.
-# github = "https://github.com/DL4DS/dl4ds_tutor"
+github = "https://github.com/DL4DS/dl4ds_tutor"
 
 # Specify a CSS file that can be used to customize the user interface.
 # The CSS file can be served from the public directory or via an external link.
@@ -85,7 +87,7 @@ custom_meta_image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/f/
 # custom_build = "./public/build"
 
 [UI.theme]
-    default = "dark"
+    default = "light"
     #layout = "wide"
     #font_family = "Inter, sans-serif"
 # Override default MUI light theme. (Check theme.ts)
@@ -115,4 +117,4 @@ custom_meta_image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/f/
         #secondary = "#BDBDBD"
 
 [meta]
-generated_by = "1.1.304"
+generated_by = "1.1.402"