Merge branch 'main' of github.com:l3vels/L3AGI

l3vels · Nov 3, 2023 · 9a86804 · 9a86804
2 parents 27725ab + 63c9041
commit 9a86804
Show file tree

Hide file tree

Showing 56 changed files with 601 additions and 880 deletions.
diff --git a/.github/workflows/deploy_server.yml b/.github/workflows/deploy_server.yml
@@ -5,14 +5,12 @@ on:
     branches:
       - main
     paths:
-      - 'apps/server/**'
-
+      - "apps/server/**"
 
 env:
   ACR_REPOSITORY: "l3-api"
   TR_DIRECTORY: "apps/server/terraform"
 
-
 jobs:
   Build:
     name: Build Application Container
@@ -24,17 +22,17 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      
-      - name: 'Build and push image'
+
+      - name: "Build and push image"
         uses: azure/docker-login@v1
         with:
           login-server: ${{ vars.ACR_REGISTRY }}
           username: ${{ secrets.AZURE_USERNAME }}
           password: ${{ secrets.AZURE_PASSWORD }}
       - run: |
-           docker build -t ${{ vars.ACR_REGISTRY }}/${{ env.ACR_REPOSITORY }}:${{ github.sha }} -f apps/server/docker/Dockerfile apps/server
-           docker push ${{ vars.ACR_REGISTRY }}/${{ env.ACR_REPOSITORY }}:${{ github.sha }}
-           echo "TAG_NAME=${{ github.sha }}" >> $GITHUB_ENV
+          docker build -t ${{ vars.ACR_REGISTRY }}/${{ env.ACR_REPOSITORY }}:${{ github.sha }} -f apps/server/docker/Dockerfile apps/server
+          docker push ${{ vars.ACR_REGISTRY }}/${{ env.ACR_REPOSITORY }}:${{ github.sha }}
+          echo "TAG_NAME=${{ github.sha }}" >> $GITHUB_ENV
 
   Init_dev:
     runs-on: ubuntu-latest
@@ -44,15 +42,15 @@ jobs:
       ENVIRONMENT: dev
     defaults:
       run:
-        shell: bash        
+        shell: bash
     steps:
       - name: Checkout
         uses: actions/[email protected]
-        
+
       - name: Change file name
-        run: | 
-          mv ${{ github.workspace }}/${{ env.TR_DIRECTORY }}/varfiles/${{ env.ENVIRONMENT  }}.tfvars  ${{ github.workspace }}/${{ env.TR_DIRECTORY }}/${{ env.ENVIRONMENT  }}.auto.tfvars 
-      
+        run: |
+          mv ${{ github.workspace }}/${{ env.TR_DIRECTORY }}/varfiles/${{ env.ENVIRONMENT  }}.tfvars  ${{ github.workspace }}/${{ env.TR_DIRECTORY }}/${{ env.ENVIRONMENT  }}.auto.tfvars
+
       - name: Update task_container_version
         run: |
           TFVARS="${{ github.workspace }}/${{ env.TR_DIRECTORY }}/${{ env.ENVIRONMENT  }}.auto.tfvars"
@@ -93,11 +91,12 @@ jobs:
           echo -e "AWS_SECRET_ACCESS_KEY = \"${{ secrets.AWS_SECRET_ACCESS_KEY }}\"\n" >> $TFVARS
           echo -e "AWS_REGION = \"${{ vars.AWS_REGION }}\"\n" >> $TFVARS
           echo -e "AWS_S3_BUCKET = \"${{ vars.AWS_S3_BUCKET }}\"\n" >> $TFVARS
-          
+          echo -e "AUTH_TOKEN = \"${{ secrets.AUTH_TOKEN }}\"\n" >> $TFVARS
+
       - name: Display .auto.tfvars file content
         run: |
           cat ${{ github.workspace }}/${{ env.TR_DIRECTORY }}/${{ env.ENVIRONMENT  }}.auto.tfvars
-  
+
       - name: Upload file
         uses: actions/upload-artifact@v2
         with:
@@ -114,8 +113,8 @@ jobs:
       tf_vars_file: varfiles/dev.tfvars
       gh_environment: dev
       task_container_name: "l3-api"
-      task_container_version: #${{ github.sha }} temp 
-      # task_container_version: "1fe3d215f1ac2e6be5d43b4e1522b9dc43aefd36" 
+      task_container_version: #${{ github.sha }} temp
+      # task_container_version: "1fe3d215f1ac2e6be5d43b4e1522b9dc43aefd36"
       task_container_registry: ${{ vars.ACR_REGISTRY }}
     secrets:
       cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }}
@@ -156,15 +155,15 @@ jobs:
       ENVIRONMENT: prod
     defaults:
       run:
-        shell: bash        
+        shell: bash
     steps:
       - name: Checkout
         uses: actions/[email protected]
-        
+
       - name: Change file name
-        run: | 
-          mv ${{ github.workspace }}/${{ env.TR_DIRECTORY }}/varfiles/${{ env.ENVIRONMENT  }}.tfvars  ${{ github.workspace }}/${{ env.TR_DIRECTORY }}/${{ env.ENVIRONMENT  }}.auto.tfvars 
-            
+        run: |
+          mv ${{ github.workspace }}/${{ env.TR_DIRECTORY }}/varfiles/${{ env.ENVIRONMENT  }}.tfvars  ${{ github.workspace }}/${{ env.TR_DIRECTORY }}/${{ env.ENVIRONMENT  }}.auto.tfvars
+
       - name: Update task_container_version
         run: |
           TFVARS="${{ github.workspace }}/${{ env.TR_DIRECTORY }}/${{ env.ENVIRONMENT  }}.auto.tfvars"
@@ -205,17 +204,17 @@ jobs:
           echo -e "AWS_SECRET_ACCESS_KEY = \"${{ secrets.AWS_SECRET_ACCESS_KEY }}\"\n" >> $TFVARS
           echo -e "AWS_REGION = \"${{ vars.AWS_REGION }}\"\n" >> $TFVARS
           echo -e "AWS_S3_BUCKET = \"${{ vars.AWS_S3_BUCKET }}\"\n" >> $TFVARS
-          
+          echo -e "AUTH_TOKEN = \"${{ secrets.AUTH_TOKEN }}\"\n" >> $TFVARS
+
       - name: Display .auto.tfvars file content
         run: |
           cat ${{ github.workspace }}/${{ env.TR_DIRECTORY }}/${{ env.ENVIRONMENT  }}.auto.tfvars
-  
+
       - name: Upload file
         uses: actions/upload-artifact@v2
         with:
           name: my-artifact
           path: ${{ github.workspace }}/${{ env.TR_DIRECTORY  }}/${{ env.ENVIRONMENT  }}.auto.tfvars
-
 
   Plan_prod:
     needs: Init_Prod
@@ -256,4 +255,4 @@ jobs:
       azure_subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
       azure_client_id: ${{ secrets.AZURE_CLIENT_ID }}
       azure_client_secret: ${{ secrets.AZURE_CLIENT_SECRET }}
-      github_personal_access_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+      github_personal_access_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
diff --git a/.github/workflows/deploy_worker.yml b/.github/workflows/deploy_worker.yml
@@ -1,36 +1,54 @@
-name: Deploy Worker
+# Docs for the Azure Web Apps Deploy action: https://github.com/Azure/webapps-deploy
+# More GitHub Actions for Azure: https://github.com/Azure/actions
+
+name: Build and Deploy Worker
 
 on:
   push:
     branches:
       - main
-    # paths:
-    # - 'apps/worker/**'
-
-env:
-  ACR_REPOSITORY_BEAT: "l3-beat"
-  ACR_REPOSITORY_WORKER: "l3-worker"
-  TR_DIRECTORY: "apps/server/terraform"
+    paths:
+      - "apps/worker/**"
+  workflow_dispatch:
 
 jobs:
-  Build_worker:
-    name: Build Application Container
-    timeout-minutes: 15
-    runs-on: ubuntu-latest
-    outputs:
-      tag_name: ${{ steps.build-image.outputs.TAG_NAME }}
-    environment: dev
+  build:
+    runs-on: "ubuntu-latest"
+
     steps:
-      - name: Checkout
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log in to registry
+        uses: docker/login-action@v2
+        with:
+          registry: https://l3container.azurecr.io/
+          username: ${{ secrets.AzureAppService_ContainerUsername_313e76d8b0ba4933b1934140d1c24667 }}
+          password: ${{ secrets.AzureAppService_ContainerPassword_a84f243ca243499bad593759e14ed4c3 }}
+
+      - name: Build and push container image to registry
+        uses: docker/build-push-action@v3
+        with:
+          context: ./apps/worker
+          push: true
+          tags: l3container.azurecr.io/${{ secrets.AzureAppService_ContainerUsername_313e76d8b0ba4933b1934140d1c24667 }}/l3-worker:${{ github.sha }}
+          file: ./apps/worker/docker/Dockerfile
 
-      - name: "Build and push image"
-        uses: azure/docker-login@v1
+  deploy:
+    runs-on: ubuntu-latest
+    needs: build
+    environment:
+      name: "production"
+      url: ${{ steps.deploy-to-webapp.outputs.webapp-url }}
+
+    steps:
+      - name: Deploy to Azure Web App
+        id: deploy-to-webapp
+        uses: azure/webapps-deploy@v2
         with:
-          login-server: ${{ vars.ACR_REGISTRY }}
-          username: ${{ secrets.AZURE_USERNAME }}
-          password: ${{ secrets.AZURE_PASSWORD }}
-      - run: |
-          docker build -t ${{ vars.ACR_REGISTRY }}/${{ env.ACR_REPOSITORY_WORKER }}:latest -f apps/worker/docker/Dockerfile apps/worker
-          docker push ${{ vars.ACR_REGISTRY }}/${{ env.ACR_REPOSITORY_WORKER }}:latest
-          echo "TAG_NAME=${{ github.sha }}" >> $GITHUB_ENV
+          app-name: "l3-worker"
+          slot-name: "production"
+          publish-profile: ${{ secrets.AzureAppService_PublishProfile_70ddda41e65f407eb7f85b1bef8d8fc7 }}
+          images: "l3container.azurecr.io/${{ secrets.AzureAppService_ContainerUsername_313e76d8b0ba4933b1934140d1c24667 }}/l3-worker:${{ github.sha }}"
diff --git a/.github/workflows/main_l3-worker.yml b/.github/workflows/main_l3-worker.yml
diff --git a/apps/server/.env.example b/apps/server/.env.example
@@ -26,6 +26,9 @@ AWS_SECRET_ACCESS_KEY=
 AWS_REGION=
 AWS_S3_BUCKET=
 
+# Auth token for worker
+AUTH_TOKEN=secret
+
 # Optionally configure to enable GitHub Authentication
 GITHUB_CLIENT_ID=
 GITHUB_CLIENT_SECRET=

diff --git a/apps/server/config.py b/apps/server/config.py
@@ -1,4 +1,5 @@
 import os
+
 from dotenv import load_dotenv
 
 load_dotenv(override=False)
@@ -35,6 +36,8 @@ class Config:
     AWS_REGION = os.environ.get("AWS_REGION")
     AWS_S3_BUCKET = os.environ.get("AWS_S3_BUCKET")
 
+    AUTH_TOKEN = os.environ.get("AUTH_TOKEN")
+
     SENTRY_DSN = os.environ.get("SENTRY_DSN")
 
     TEST_USER_EMAIL = os.environ.get("TEST_USER_EMAIL")

diff --git a/apps/server/controllers/fine_tuning.py b/apps/server/controllers/fine_tuning.py
@@ -10,15 +10,17 @@
 from services.fine_tuning import check_fine_tuning, fine_tune_openai_model
 from typings.auth import UserAccount
 from typings.fine_tuning import FineTuningInput, FineTuningOutput
-from utils.auth import authenticate
+from utils.auth import authenticate, authenticate_by_auth_token
 from utils.fine_tuning import (convert_fine_tunings_to_fine_tuning_list,
                                convert_model_to_response)
 
 router = APIRouter()
 
 
 @router.post(
-    "/{fine_tuning_id}/check", status_code=200, response_model=FineTuningOutput
+    "/{fine_tuning_id}/check",
+    status_code=200,
+    dependencies=[Depends(authenticate_by_auth_token)],
 )
 def check_fine_tuning_status(fine_tuning_id: UUID):
     check_fine_tuning(db.session, fine_tuning_id)
@@ -93,10 +95,12 @@ def update_fine_tuning(
         raise HTTPException(status_code=404, detail="Fine-tuning not found")
 
 
-@router.get("/pending", response_model=List[FineTuningOutput])
-def get_pending_fine_tunings(
-    auth: UserAccount = Depends(authenticate),
-) -> List[FineTuningOutput]:
+@router.get(
+    "/pending",
+    response_model=List[FineTuningOutput],
+    dependencies=[Depends(authenticate_by_auth_token)],
+)
+def get_pending_fine_tunings() -> List[FineTuningOutput]:
     """
     Get all pending fine-tunings for worker.
 

diff --git a/apps/server/controllers/schedule.py b/apps/server/controllers/schedule.py
@@ -5,20 +5,22 @@
 
 from exceptions import ScheduleNotFoundException
 from models.schedule import ScheduleModel
-from typings.auth import UserAccount
-from typings.schedule import (
-    ScheduleConfigInput,
-    ScheduleWithConfigsOutput,
-    ScheduleStatus,
-)
-from utils.auth import authenticate
-from utils.schedule import convert_model_to_response, convert_schedules_to_schedule_list
 from services.schedule import execute_scheduled_run
+from typings.auth import UserAccount
+from typings.schedule import (ScheduleConfigInput, ScheduleStatus,
+                              ScheduleWithConfigsOutput)
+from utils.auth import authenticate, authenticate_by_auth_token
+from utils.schedule import (convert_model_to_response,
+                            convert_schedules_to_schedule_list)
 
 router = APIRouter()
 
 
-@router.post("/{schedule_id}/run", status_code=200)
+@router.post(
+    "/{schedule_id}/run",
+    status_code=200,
+    dependencies=[Depends(authenticate_by_auth_token)],
+)
 def run_schedule(schedule_id: str):
     schedule = ScheduleModel.get_schedule_by_id(db, schedule_id, None)
 
@@ -28,9 +30,11 @@ def run_schedule(schedule_id: str):
     if schedule.status == ScheduleStatus.PROCESSING.value:
         raise HTTPException(status_code=400, detail="Schedule already is processing")
 
-    execute_scheduled_run(db.session, schedule)
-
-    return {"message": "Schedule run successfully"}
+    try:
+        execute_scheduled_run(db.session, schedule)
+        return {"message": "Schedule run successfully"}
+    except Exception as err:
+        raise HTTPException(status_code=500, detail=str(err))
 
 
 @router.post("", status_code=201, response_model=ScheduleWithConfigsOutput)
@@ -124,7 +128,7 @@ def get_schedules(
     return convert_schedules_to_schedule_list(db_schedules)
 
 
-@router.get("/due")
+@router.get("/due", dependencies=[Depends(authenticate_by_auth_token)])
 def get_due_schedules():
     schedules = ScheduleModel.get_due_schedules(db.session)
     return convert_schedules_to_schedule_list(schedules)

diff --git a/apps/server/main.py b/apps/server/main.py
@@ -64,8 +64,6 @@ class Mutation(AccountMutation, UserMutation):
 
 # Base.metadata.create_all(bind=engine)
 
-# if Config.ENV != "local"
-
 
 # origins = [
 #     "http://localhost:3000",