Merge pull request #370 from AutomatingSciencePipeline/358-set-max-nu…

…mber-of-items-on-dashboard-05

Refactor to remove subscribe

apps/frontend/app/components/flows/ViewExperiment/ExperimentListing.tsx

@@ -7,16 +7,16 @@ import { Timestamp } from 'mongodb';
 import { updateExperimentNameById } from '../../../../lib/mongodb_funcs';
 
 export interface ExperimentListingProps {
-	projectinit: ExperimentData;
+	projectData: ExperimentData;
 	onCopyExperiment: (experimentId: string) => void;
 	onDownloadResults: (experimentId: string) => Promise<void>;
 	onDownloadProjectZip: (experimentId: string) => Promise<void>;
 	onDeleteExperiment: (experimentId: string) => void;
 }
 
 
-export const ExperimentListing = ({ projectinit, onCopyExperiment, onDownloadResults, onDownloadProjectZip, onDeleteExperiment }: ExperimentListingProps) => {
-	const [project, setProject] = useState<ExperimentData>(projectinit);
+export const ExperimentListing = ({ projectData: projectData, onCopyExperiment, onDownloadResults, onDownloadProjectZip, onDeleteExperiment }: ExperimentListingProps) => {
+	const [project, setProject] = useState<ExperimentData>(projectData);
 
 	const [busyDownloadingResults, setBusyDownloadingResults] = useState<boolean>(false);
 	const [busyDownloadingZip, setBusyDownloadingZip] = useState<boolean>(false);
@@ -31,10 +31,10 @@ export const ExperimentListing = ({ projectinit, onCopyExperiment, onDownloadRes
 	const expectedFinishTime = experimentInProgress ? new Date(project['startedAtEpochMillis'] + expectedTimeToRun * 60000) : null;
 	// 60000 milliseconds in a minute  // Set to null if the experiment is not in progress
 
-	const [projectName, setProjectName] = useState(projectinit.name); // New state for edited project name
+	const [projectName, setProjectName] = useState(projectData.name); // New state for edited project name
 	const [isEditing, setIsEditing] = useState(false);
 	const [editingCanceled, setEditingCanceled] = useState(false); // New state for tracking editing cancellation
-	const [originalProjectName, setOriginalProjectName] = useState(projectinit.name); // State to store the original project name
+	const [originalProjectName, setOriginalProjectName] = useState(projectData.name); // State to store the original project name
 
 	const [isDeleteModalOpen, setDeleteModalOpen] = useState(false);
 
@@ -65,16 +65,15 @@ export const ExperimentListing = ({ projectinit, onCopyExperiment, onDownloadRes
 			setProjectName(originalProjectName); // Revert to the original name
 			setEditingCanceled(true);
 		} else {
-			const eventSource = new EventSource(`/api/experiments/subscribe?expId=${project.expId}`);
-			eventSource.onmessage = (event) => {
-				if (event.data !== 'heartbeat' && event.data) {
-					setProject(JSON.parse(event.data) as ExperimentData);
-				}
-
-			}
+			// Do nothing here?
 		}
 	}, [editingCanceled, originalProjectName, project.expId]);
 
+	//Update the project when data is changed
+	useEffect(() => {
+		setProject(projectData);
+	}, [projectData]);
+
 
 	const handleKeyUp = (e) => {
 		if (e.key === 'Enter') {

apps/frontend/app/dashboard/page.tsx

@@ -765,7 +765,7 @@ const ExperimentList = ({ experiments, onCopyExperiment, onDeleteExperiment }: E
 						className='relative pl-4 pr-6 py-5 hover:bg-gray-50 sm:py-6 sm:pl-6 lg:pl-8 xl:pl-6'
 					>
 						<ExperimentListing
-							projectinit={project}
+							projectData={project}
 							onCopyExperiment={onCopyExperiment}
 							onDownloadResults={downloadExperimentResults}
 							onDownloadProjectZip={downloadExperimentProjectZip}

apps/frontend/frontend.Dockerfile

@@ -35,15 +35,6 @@ WORKDIR /app
 COPY --from=deps /app/node_modules ./node_modules
 COPY . .
 
-
-# Setup env vars
-ARG MONGODB_PORT
-ARG MONGODB_USERNAME
-ARG MONGODB_PASSWORD
-ENV MONGODB_PORT=${MONGODB_PORT}
-ENV MONGODB_USERNAME=${MONGODB_USERNAME}
-ENV MONGODB_PASSWORD=${MONGODB_PASSWORD}
-
 RUN npm run build
 
 FROM base AS runner

apps/frontend/lib/mongodb.ts

@@ -4,9 +4,9 @@ import { getEnvVar } from '../utils/env';
 
 // Adapted from https://github.com/vercel/next.js/tree/canary/examples/with-mongodb
 
-const MONGODB_PORT = getEnvVar('MONGODB_PORT');
-const USERNAME = getEnvVar('MONGODB_USERNAME');
-const PASSWORD = getEnvVar('MONGODB_PASSWORD');
+const MONGODB_PORT = process.env.MONGODB_PORT || '1234';
+const USERNAME = process.env.MONGODB_USERNAME || 'user';
+const PASSWORD = process.env.MONGODB_PASSWORD || 'pass';
 
 const MONGODB_URI = `mongodb://${USERNAME}:${PASSWORD}@glados-service-mongodb:${MONGODB_PORT}`;
 const MONGODB_OPTIONS = {};

apps/frontend/pages/api/experiments/listen.tsx

@@ -33,7 +33,7 @@ export default async function handler(req, res) {
             }
         }
     ];
-    const options = {};
+    const options = { fullDocument: "updateLookup" };
     const changeStream = experimentsCollection.watch(pipeline, options);
 
     // Set up real-time streaming of changes to the client using SSE

development_scripts/local/setup_local.py

@@ -0,0 +1,129 @@
+import base64
+import os
+import subprocess
+import sys
+import pathlib
+import threading
+
+def setup(args):
+    if len(args) == 0:
+        print("Please provide at least one of the following arguments: frontend, backend, runner, all")
+        return
+
+    if not pathlib.Path().resolve().name == "Monorepo":
+        print("Please run this script from the root of the Monorepo")
+        return
+
+    print("Setting up local environment")
+
+    # SET YOUR DOCKER HUB USERNAME HERE!
+    docker_hub_username = "YOUR_DOCKER_HUB_USERNAME"
+
+    # SET KEYCLOAK INFO HERE!
+    # Ask Riley how to set this up
+    keycloak_url = "http://glados-w0.csse.rose-hulman.edu:8080/realms/master"
+    keycloak_client_id = "YOUR_KEYCLOAK_CLIENT_ID"
+    keycloak_client_secret = "YOUR_KEYCLOAK_CLIENT_SECRET"
+
+    if "frontend" in args:
+        print("Building and pushing frontend image")
+        # open a terminal and run the following command
+        os.system(f"docker build -t {docker_hub_username}/glados-frontend:main -f ./apps/frontend/frontend.Dockerfile ./apps/frontend")
+        os.system(f"docker push {docker_hub_username}/glados-frontend:main")
+
+    if "backend" in args:
+        print("Building and pushing backend image")
+        # open a terminal and run the following command
+        os.system(f"docker build -t {docker_hub_username}/glados-backend:main -f ./apps/backend/backend.Dockerfile ./apps/backend")
+        os.system(f"docker push {docker_hub_username}/glados-backend:main")
+
+    if "runner" in args:
+        print("Building and pushing runner image")
+        # open a terminal and run the following command
+        os.system(f"docker build -t {docker_hub_username}/glados-runner:main -f ./apps/runner/runner.Dockerfile ./apps/runner")
+        os.system(f"docker push {docker_hub_username}/glados-runner:main")
+
+    if "all" in args:
+        print("Building and pushing all images")
+        # open a terminal and run the following command
+        os.system(f"docker build -t {docker_hub_username}/glados-frontend:main -f ./apps/frontend/frontend.Dockerfile ./apps/frontend")
+        os.system(f"docker push {docker_hub_username}/glados-frontend:main")
+        os.system(f"docker build -t {docker_hub_username}/glados-backend:main -f ./apps/backend/backend.Dockerfile ./apps/backend")
+        os.system(f"docker push {docker_hub_username}/glados-backend:main")
+        os.system(f"docker build -t {docker_hub_username}/glados-runner:main -f ./apps/runner/runner.Dockerfile ./apps/runner")
+        os.system(f"docker push {docker_hub_username}/glados-runner:main")
+
+    print("Setting up kubernetes")
+
+    os.system("python3 kubernetes_init\\init.py --hard")
+
+    # in a new thread we need to run the following command
+    # minikube service glados-frontend --url
+    # we then need to use that url to setup some secret stuff, and then redeploy while keeping that open
+
+    def run_minikube_service():
+        process = subprocess.Popen(["minikube", "service", "glados-frontend", "--url"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+        for line in process.stdout: # type: ignore
+            print(line)
+            url = line.strip()
+            port = url.split(":")[-1]
+
+            # go into the kubernetes_init/secrets folder and then update the auth url line to be
+            # http://localhost:<port>/api/auth
+            # update the line with AUTH_URL
+            # values are base64 encoded
+            f = open("kubernetes_init/kubernetes_secrets/secret.yaml", "r")
+            lines = f.readlines()
+            f.close()
+
+            for i in range(len(lines)):
+                if "AUTH_URL" in lines[i] or "AUTH_REDIRECT_PROXY_URL" in lines[i]:
+                    # base 64 encode the url
+                    b64Url = base64.b64encode(f"http://localhost:{port}/api/auth".encode()).decode()
+                    lines[i] = f"  AUTH_URL: {b64Url}\n"
+                    break
+
+                if "AUTH_KEYCLOAK_ISSUER" in lines[i]:
+                    b64Url = base64.b64encode(keycloak_url.encode()).decode()
+                    lines[i] = f"  AUTH_KEYCLOAK_ISSUER: {b64Url}\n"
+
+                if "AUTH_KEYCLOAK_ID" in lines[i]:
+                    b64Id = base64.b64encode(keycloak_client_id.encode()).decode()
+                    lines[i] = f"  AUTH_KEYCLOAK_ID: {b64Id}\n"
+
+                if "AUTH_KEYCLOAK_SECRET" in lines[i]:
+                    b64Secret = base64.b64encode(keycloak_client_secret.encode()).decode()
+                    lines[i] = f"  AUTH_KEYCLOAK_SECRET: {b64Secret}\n"
+
+            # update the file
+            f = open("kubernetes_init/kubernetes_secrets/secret.yaml", "w")
+            f.writelines(lines)
+            f.close()
+
+            # apply the new secrets
+            os.system("python3 kubernetes_init\\init.py")
+
+            print(f"Frontend is now running at: http://localhost:{port}")        
+
+            break
+    if "skip" in args:
+            print("Skipping redeploy")
+            print("Frontend is available where it was before...")
+            return
+
+    # try to delete the old service
+    # do this so that the old minikube service dies
+    os.system("kubectl delete svc glados-frontend")
+    os.system("kubectl expose deployment glados-frontend --type LoadBalancer --port 80 --target-port 3000")
+
+    thread = threading.Thread(target=run_minikube_service)
+    thread.start()
+
+
+    print("Setup complete")
+
+
+
+if __name__ == "__main__":
+    args = sys.argv[1:]
+    setup(args)

docs/docs/tutorial/local_testing.md

@@ -199,8 +199,8 @@ Which will show something like
 
 ```bash
 NAME                                        READY   STATUS    RESTARTS       AGE
-glados-backend-687fc6b7ff-dld2p    1/1     Running   0              74s
-glados-frontend-5f575b99b7-9q9ml   1/1     Running   0              74s
+glados-backend-687fc6b7ff-dld2p             1/1     Running   0              74s
+glados-frontend-5f575b99b7-9q9ml            1/1     Running   0              74s
 glados-mongodb-0                            1/1     Running   1 (2m9s ago)   20d
 glados-mongodb-1                            1/1     Running   1 (2m9s ago)   20d
 glados-mongodb-arbiter-0                    1/1     Running   2 (20d ago)    20d
@@ -220,3 +220,32 @@ This will then show the image information. Make sure this points to your docker
 In order to update the runner image, go to the apps/backend folder, and update the image in job-runner.yaml following the steps above.
 
 Now you can use locally built images to run GLADOS!
+
+## Prebuilt Script for Docker Image Management
+
+Due to the complexity of getting Minikube to behave, I have created a python script to run the needed commands for you. 
+
+From the root of the Monorepo run the command:
+
+```bash
+python3 .\development_scripts\local\setup_local.py <args>
+```
+
+You can provide arguments for which elements of the project you would like to build.
+
+Options are: frontend, backend, runner, all
+
+In the python3 file you will need to set a couple of values to make sure that it is setup for your environment. Update those values and run the python script with the pieces you would like to build and push.
+
+Note: You still need to make sure to follow the steps above for changing the image which you are running the cluster from.
+
+After running the python script you will see something like:
+
+```bash
+Frontend is now running at: http://localhost:64068
+```
+
+Opening that link will bring you to a local version of GLADOS.
+
+!!!Warning
+    With the local version of GLADOS being HTTP you may have weird networking issues due to the max number of connections to an HTTP/1.1 host. This will be fixed in a later update.