diff --git a/apps/backend/app.py b/apps/backend/app.py index 73fcfc66..cd8b67c8 100644 --- a/apps/backend/app.py +++ b/apps/backend/app.py @@ -102,8 +102,8 @@ def check_mongo(): @flaskApp.get("/downloadExpFile") def download_exp_file(): try: - experiment_id = request.args.get('expId', default='', type=str) - file_data = download_experiment_file(experiment_id, mongoClient) + file_id = request.args.get('fileId', default='', type=str) + file_data = download_experiment_file(file_id, mongoClient) file_stream = io.BytesIO(file_data) return send_file(file_stream, as_attachment=True, download_name="experiment_file", mimetype="application/octet-stream") except Exception: diff --git a/apps/backend/modules/mongo.py b/apps/backend/modules/mongo.py index ccd3eb6f..747b21b0 100644 --- a/apps/backend/modules/mongo.py +++ b/apps/backend/modules/mongo.py @@ -70,12 +70,12 @@ def insertExperiments(): # keep trying check_insert_default_experiments(mongoClient) -def download_experiment_file(expId: str, mongoClient: pymongo.MongoClient): +def download_experiment_file(file_id: str, mongoClient: pymongo.MongoClient): # we are going to have to get the binary data from mongo here # setup the bucket db = mongoClient["gladosdb"] bucket = GridFSBucket(db, bucket_name='fileBucket') - files = bucket.find({"metadata.expId": expId}).to_list() + files = bucket.find({"_id": ObjectId(file_id)}).to_list() # type: ignore num_files = 0 file_name = "" for file in files: diff --git a/apps/frontend/app/components/auth/SignUpModal.tsx b/apps/frontend/app/components/auth/SignUpModal.tsx index cefcb129..27d52da3 100644 --- a/apps/frontend/app/components/auth/SignUpModal.tsx +++ b/apps/frontend/app/components/auth/SignUpModal.tsx @@ -67,9 +67,9 @@ export const SignUpModal = ({ afterSignUp }) => { - {/* For dev testing!!! */} - {/* */} + {/* For dev testing!!! */} + {/* */}
diff --git a/apps/frontend/app/components/flows/AddExperiment/NewExperiment.tsx b/apps/frontend/app/components/flows/AddExperiment/NewExperiment.tsx index 072e533f..30491118 100644 --- a/apps/frontend/app/components/flows/AddExperiment/NewExperiment.tsx +++ b/apps/frontend/app/components/flows/AddExperiment/NewExperiment.tsx @@ -11,9 +11,11 @@ import { ParamStep } from './stepComponents/ParamStep'; import { PostProcessStep } from './stepComponents/PostProcessStep'; import { ConfirmationStep } from './stepComponents/ConfirmationStep'; import { DumbTextArea } from './stepComponents/DumbTextAreaStep'; -import { DB_COLLECTION_EXPERIMENTS } from '../../../../lib/db'; +import { DB_COLLECTION_EXPERIMENTS, submitExperiment } from '../../../../lib/db'; -import { getDocumentFromId } from '../../../../lib/mongodb_funcs'; +import { getDocumentFromId, updateLastUsedDateFile } from '../../../../lib/mongodb_funcs'; +import { useSession } from 'next-auth/react'; +import toast, { Toaster } from 'react-hot-toast'; const DEFAULT_TRIAL_TIMEOUT_SECONDS = 5 * 60 * 60; // 5 hours in seconds @@ -63,6 +65,8 @@ const Steps = ({ steps }) => { const NewExperiment = ({ formState, setFormState, copyID, setCopyId, ...rest }) => { + const { data: session } = useSession(); + const form = useForm({ // TODO make this follow the schema as closely as we can initialValues: { @@ -124,11 +128,7 @@ const NewExperiment = ({ formState, setFormState, copyID, setCopyId, ...rest }) const [status, setStatus] = useState(0); const [id, setId] = useState(null); - const onDropComplete = () => { - setFormState(-1); - localStorage.removeItem('ID'); - setStatus(FormStates.Info); - }; + const [fileId, setFileId] = useState(); useLayoutEffect(() => { if (formState === FormStates.Info) { @@ -142,123 +142,154 @@ const NewExperiment = ({ formState, setFormState, copyID, setCopyId, ...rest }) }, [formState]); // TODO adding 'form' causes an update loop return ( - - setFormState(0)} - > -
- +
+ + + setFormState(0)} + > +
+ -
- -
-
{ - })} - > -
-
-
- { - return { - id: idx + 1, - name: step, - status: status < idx ? 'upcoming' : 'complete', - }; - } - )} - /> +
+ +
+ { + })} + > +
+
+
+ { + return { + id: idx + 1, + name: step, + status: status < idx ? 'upcoming' : 'complete', + }; + } + )} + /> +
-
- {/*
*/} - {status === FormStates.Info ? ( - - ) : status === FormStates.DumbTextArea ? ( - - ) : status === FormStates.Params ? ( - {fields} - ) : status === FormStates.ProcessStep ? ( - {fields} - ) : status === FormStates.Confirmation ? ( - - ) : ( - - )} + {/*
*/} + {status === FormStates.Info ? ( + + ) : status === FormStates.DumbTextArea ? ( + + ) : status === FormStates.Params ? ( + {fields} + ) : status === FormStates.ProcessStep ? ( + {fields} + ) : status === FormStates.Confirmation ? ( + + ) : ( + + )} -
-
-
- - { - form.setFieldValue('verbose', !form.values.verbose); - }} - /> -
- + - {!(status === FormStates.Dispatch) && } + { + type: 'button', + onClick: () => setStatus(status + 1), + })} + > + {status === FormStates.Dispatch ? 'Dispatch' : 'Next'} + +
-
- -
- + +
+ +
-
-
- +
+ +
); }; diff --git a/apps/frontend/app/components/flows/AddExperiment/stepComponents/DispatchStep.tsx b/apps/frontend/app/components/flows/AddExperiment/stepComponents/DispatchStep.tsx index 980c194c..61e59015 100644 --- a/apps/frontend/app/components/flows/AddExperiment/stepComponents/DispatchStep.tsx +++ b/apps/frontend/app/components/flows/AddExperiment/stepComponents/DispatchStep.tsx @@ -1,12 +1,12 @@ 'use client' import { Dropzone, DropzoneProps } from '@mantine/dropzone'; -import { submitExperiment } from '../../../../../lib/db'; import { Group, Text } from '@mantine/core'; import { useSession } from "next-auth/react"; import { Upload, FileCode } from 'tabler-icons-react'; -import { useState } from 'react'; +import { useEffect, useState } from 'react'; +import { getRecentFiles } from '../../../../../lib/mongodb_funcs'; import { on } from 'events'; const SUPPORTED_FILE_TYPES = { @@ -18,93 +18,132 @@ const SUPPORTED_FILE_TYPES = { 'application/x-elf': [], // does nothing atm, from what I can tell }; -export const DispatchStep = ({ id, form, onDropComplete, ...props }) => { +export const DispatchStep = ({ id, form, updateId, ...props }) => { const { data: session } = useSession(); const [loading, setLoading] = useState(false); - const onDropFile = (files: Parameters[0]) => { + const [userFiles, setUserFiles] = useState([]); + + const [selectedFile, setSelectedFile] = useState(""); + + useEffect(() => { + getRecentFiles(session?.user?.id!).then((files) => { + setUserFiles(files); + }).catch((error) => console.error("Error fetching files:", error)); + }, [session?.user?.id]); + + + const onDropFile = async (files: Parameters[0]) => { setLoading(true); - submitExperiment(form.values, session?.user?.id as string).then(async (json) => { - const expId = json['id']; - const formData = new FormData(); - formData.set("file", files[0]); - formData.set("expId", expId); - const uploadResponse = await fetch('/api/files/uploadFile', { - method: 'POST', - credentials: 'same-origin', - body: formData - }); - if (uploadResponse.ok) { - console.log(`Handing experiment ${expId} to the backend`); - const response = await fetch(`/api/experiments/start/${expId}`, { - method: 'POST', - headers: new Headers({ 'Content-Type': 'application/json' }), - credentials: 'same-origin', - body: JSON.stringify({ id: expId }), - }); - if (response.ok) { - console.log('Response from backend received', response); - } else { - const responseText = await response.text(); - console.log('Upload failed', responseText, response); - throw new Error(`Upload failed: ${response.status}: ${responseText}`); - } - } else { - throw new Error('Failed to upload experiment file to the backend server, is it running?'); - } - }).catch((error) => { - console.log('Error uploading experiment: ', error); - alert(`Error uploading experiment: ${error.message}`); - }).finally(() => { - setLoading(false); - onDropComplete(); + const formData = new FormData(); + formData.set("file", files[0]); + formData.set("userId", session?.user?.id!); + const uploadFileResponse = await fetch('/api/files/uploadFile', { + method: 'POST', + credentials: 'same-origin', + body: formData }); + if (uploadFileResponse.ok) { + const json = await uploadFileResponse.json(); + const fileId = json['fileId']; + const fileName = json['fileName']; + updateId(fileId); + setSelectedFile(fileName); + setLoading(false); + } + else { + const json = await uploadFileResponse.json(); + console.log(`Failed to upload file: ${json['message']}`); + } }; const MAXIMUM_SIZE_BYTES = 3 * 1024 ** 2; return ( - { - console.log('File rejection details', rejections); - const uploadedType = rejections[0]?.file?.type; - alert(`Rejected:\n${rejections[0]?.errors[0]?.message}\nYour file was of type: ${uploadedType ? uploadedType : 'Unknown'}\nCheck the console for more details.`); - }} - maxSize={MAXIMUM_SIZE_BYTES} - maxFiles={1} - className='flex-1 flex flex-col justify-center m-4 items-center' - loading={loading} - // accept={SUPPORTED_FILE_TYPES} - > - - - - - - {/* For some reason (seems to be happening on React itself's side?) the dropzone is claiming to reject files even if the file's mime +
+
+ { + console.log('File rejection details', rejections); + const uploadedType = rejections[0]?.file?.type; + alert(`Rejected:\n${rejections[0]?.errors[0]?.message}\nYour file was of type: ${uploadedType ? uploadedType : 'Unknown'}\nCheck the console for more details.`); + }} + maxSize={MAXIMUM_SIZE_BYTES} + maxFiles={1} + className='justify-center m-4 items-center h-full' + loading={loading} + // accept={SUPPORTED_FILE_TYPES} + > + + + + + + {/* For some reason (seems to be happening on React itself's side?) the dropzone is claiming to reject files even if the file's mime type is included in the accept list we pass it. Works for images, when changed to be images, but not our stuff. Check browser console and you can see that the file object's type really does match what we have in our list!*/} - - {/* */} - - - - + + {/* */} + + + + + +
+ + Upload your project executable. + + + Drag-and-drop, or click here to open a file picker. + + + {/* Supported: {[...new Set(Object.values(SUPPORTED_FILE_TYPES).flat())].join(', ')} */} + Supporting .py, .jar and binary executables + +
+ + +
+
+

Recent Files

+ + + + + + + + + + {userFiles.map((file) => ( + + + + + + ))} + +
SelectFilenameLast Use Date
+ + {file.filename} + {new Date(file.metadata.lastUsedDate).toLocaleString()} +
+
+
+ { + selectedFile ? 'Current selected file: ' + selectedFile : <> + } +
+
-
- - Upload your project executable. - - - Drag-and-drop, or click here to open a file picker. - - - {/* Supported: {[...new Set(Object.values(SUPPORTED_FILE_TYPES).flat())].join(', ')} */} - Supporting .py, .jar and binary executables - -
- - ); }; diff --git a/apps/frontend/app/dashboard/page.tsx b/apps/frontend/app/dashboard/page.tsx index ea25c4a5..66f785e0 100644 --- a/apps/frontend/app/dashboard/page.tsx +++ b/apps/frontend/app/dashboard/page.tsx @@ -24,6 +24,7 @@ import { Toggle } from '../components/Toggle'; import { QueueResponse } from '../../pages/api/queue'; import { deleteDocumentById } from '../../lib/mongodb_funcs'; import { signOut, useSession } from "next-auth/react"; +import toast, { Toaster } from 'react-hot-toast'; const navigation = [{ name: 'Admin', href: '#', current: false }]; const userNavigation = [ @@ -265,6 +266,7 @@ export default function DashboardPage() { return ( <> + {/* Background color split screen for large screens */}
{ // deleteExperiment(experimentId); - deleteDocumentById(experimentId).catch((reason) => { + deleteDocumentById(experimentId).then(() => { + toast.success("Deleted experiment!", {duration: 1500}); + }).catch((reason) => { + toast.error(`Failed delete, reason: ${reason}`, {duration: 1500}); console.log(`Failed delete, reason: ${reason}`); - }) + }); }} />
{/* Activity feed */} diff --git a/apps/frontend/app/layout.tsx b/apps/frontend/app/layout.tsx index e9b3bde6..4030c572 100644 --- a/apps/frontend/app/layout.tsx +++ b/apps/frontend/app/layout.tsx @@ -1,6 +1,7 @@ import '../styles/globals.css'; import '../styles/experimentListing.css'; import RouteHandler from './RouteHandler'; +import Head from 'next/head'; export default function RootLayout({ children, @@ -9,7 +10,9 @@ export default function RootLayout({ }) { return ( - + + GLADOS + {children} diff --git a/apps/frontend/lib/db.ts b/apps/frontend/lib/db.ts index bcf42ac6..72b5f1d9 100644 --- a/apps/frontend/lib/db.ts +++ b/apps/frontend/lib/db.ts @@ -5,12 +5,13 @@ import { ResultsCsv, ProjectZip } from '../lib/mongodb_types'; export const DB_COLLECTION_EXPERIMENTS = 'Experiments'; // test -export const submitExperiment = async (values: Partial, userId: string) => { +export const submitExperiment = async (values: Partial, userId: string, fileId: string) => { values.creator = userId; values.created = Date.now(); values.finished = false; values.estimatedTotalTimeMinutes = 0; values.totalExperimentRuns = 0; + values.file = fileId; const response = await fetch(`/api/experiments/storeExp`, { method: "POST", diff --git a/apps/frontend/lib/mongodb_funcs.ts b/apps/frontend/lib/mongodb_funcs.ts index ebddfd28..7af8ea5f 100644 --- a/apps/frontend/lib/mongodb_funcs.ts +++ b/apps/frontend/lib/mongodb_funcs.ts @@ -1,5 +1,5 @@ 'use server'; -import { ObjectId } from "mongodb"; +import { GridFSBucket, ObjectId } from "mongodb"; import clientPromise, { DB_NAME, COLLECTION_EXPERIMENTS } from "./mongodb"; export async function getDocumentFromId(expId: string) { @@ -12,8 +12,24 @@ export async function getDocumentFromId(expId: string) { return Promise.reject(`Could not find document with id: ${expId}`); } + const expInfo = { + hyperparameters: Array.isArray(expDoc.hyperparameters) ? expDoc.hyperparameters : [], + name: expDoc.name || '', + description: expDoc.description || '', + trialExtraFile: expDoc.trialExtraFile || '', + trialResult: expDoc.trialResult || '', + verbose: expDoc.verbose || false, + workers: expDoc.workers || 0, + scatter: expDoc.scatter || '', + dumbTextArea: expDoc.dumbTextArea || '', + scatterIndVar: expDoc.scatterIndVar || '', + scatterDepVar: expDoc.scatterDepVar || '', + timeout: expDoc.timeout || 0, + keepLogs: expDoc.keepLogs || false, + }; + //just return the document - return expDoc; + return expDoc } export async function deleteDocumentById(expId: string) { @@ -43,3 +59,45 @@ export async function updateExperimentNameById(expId: string, newExpName: string return Promise.resolve(); } + +export async function getRecentFiles(userId: string) { + 'use server'; + const client = await clientPromise; + const db = client.db(DB_NAME); + const bucket = new GridFSBucket(db, { bucketName: 'fileBucket' }); + + const userFiles = await bucket.find({ "metadata.userId": userId }) + .sort({ "metadata.lastUsedDate": -1 }) + .limit(5) + .toArray(); + + // Transform the data to be JSON-serializable + const serializedFiles = userFiles.map(file => ({ + _id: file._id.toString(), // Convert ObjectId to string + length: file.length, + chunkSize: file.chunkSize, + uploadDate: file.uploadDate.toISOString(), // Convert Date to ISO string + filename: file.filename, + metadata: file.metadata, // Assuming metadata is already serializable + })); + + return serializedFiles; +} + +export async function updateLastUsedDateFile(fileId: string) { + 'use server'; + const client = await clientPromise; + const db = client.db(DB_NAME); + const bucket = new GridFSBucket(db, { bucketName: 'fileBucket' }); + + const file = await bucket.find({ _id: new ObjectId(fileId) }).toArray(); + if (file.length === 0) { + return; + } + + await db.collection('fileBucket.files').updateOne( + { _id: new ObjectId(fileId) }, + { $set: { 'metadata.lastUsedDate': new Date() } } + ); +} + diff --git a/apps/frontend/package-lock.json b/apps/frontend/package-lock.json index 0fddb751..279a4426 100644 --- a/apps/frontend/package-lock.json +++ b/apps/frontend/package-lock.json @@ -35,6 +35,7 @@ "react": "^18.3.1", "react-beautiful-dnd": "^13.1.1", "react-dom": "^18.3.1", + "react-hot-toast": "^2.4.1", "react-icons": "^4.6.0", "tabler-icons-react": "^1.55.0", "uuid": "^9.0.0" @@ -4807,6 +4808,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/goober": { + "version": "2.1.16", + "resolved": "https://registry.npmjs.org/goober/-/goober-2.1.16.tgz", + "integrity": "sha512-erjk19y1U33+XAMe1VTvIONHYoSqE4iS7BYUZfHaqeohLmnC0FdxEh7rQU+6MZ4OajItzjZFSRtVANrQwNq6/g==", + "license": "MIT", + "peerDependencies": { + "csstype": "^3.0.10" + } + }, "node_modules/gopd": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", @@ -6917,6 +6927,22 @@ "react": ">= 16.8 || 18.0.0" } }, + "node_modules/react-hot-toast": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/react-hot-toast/-/react-hot-toast-2.4.1.tgz", + "integrity": "sha512-j8z+cQbWIM5LY37pR6uZR6D4LfseplqnuAO4co4u8917hBUvXlEqyP1ZzqVLcqoyUesZZv/ImreoCeHVDpE5pQ==", + "license": "MIT", + "dependencies": { + "goober": "^2.1.10" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "react": ">=16", + "react-dom": ">=16" + } + }, "node_modules/react-icons": { "version": "4.10.1", "resolved": "https://registry.npmjs.org/react-icons/-/react-icons-4.10.1.tgz", diff --git a/apps/frontend/package.json b/apps/frontend/package.json index 822032da..843381cd 100644 --- a/apps/frontend/package.json +++ b/apps/frontend/package.json @@ -38,6 +38,7 @@ "react": "^18.3.1", "react-beautiful-dnd": "^13.1.1", "react-dom": "^18.3.1", + "react-hot-toast": "^2.4.1", "react-icons": "^4.6.0", "tabler-icons-react": "^1.55.0", "uuid": "^9.0.0" diff --git a/apps/frontend/pages/api/files/uploadFile.tsx b/apps/frontend/pages/api/files/uploadFile.tsx index 435a71d2..59dca238 100644 --- a/apps/frontend/pages/api/files/uploadFile.tsx +++ b/apps/frontend/pages/api/files/uploadFile.tsx @@ -1,7 +1,6 @@ import clientPromise, { DB_NAME, COLLECTION_RESULTS_CSVS, COLLECTION_EXPERIMENT_FILES } from '../../../lib/mongodb'; import { NextApiHandler, NextApiRequest } from 'next'; import { GridFSBucket } from 'mongodb'; -import { Readable, Writable } from 'stream'; import formidable, { Fields, Files } from "formidable"; import fs from "fs"; @@ -14,7 +13,7 @@ export const config = { // Helper function to parse form data with formidable const parseForm = async (req: NextApiRequest): Promise<{ fields: Fields; files: Files }> => new Promise((resolve, reject) => { - const form = formidable({ keepExtensions: true }); + const form = formidable({ keepExtensions: true, hashAlgorithm: "sha256" }); form.parse(req, (err, fields, files) => { if (err) reject(err); resolve({ fields, files }); @@ -25,9 +24,9 @@ const parseForm = async (req: NextApiRequest): Promise<{ fields: Fields; files: const mongoFileUploader: NextApiHandler = async (req, res) => { if (req.method === 'POST') { const { fields, files } = await parseForm(req); - const expId = Array.isArray(fields.expId) ? fields.expId[0] : fields.expId; - - if (!files.file || !expId) { + const userId = Array.isArray(fields.userId) ? fields.userId[0] : fields.userId; + + if (!files.file || !userId) { return res.status(400).json({ response: "Not enough arguments!" } as any); } @@ -37,23 +36,35 @@ const mongoFileUploader: NextApiHandler = async (req, res) => { const bucket = new GridFSBucket(db, { bucketName: 'fileBucket' }); const file = Array.isArray(files.file) ? files.file[0] : files.file; + //Try to find that file hash in the database + const identicalFile = bucket.find({ "metadata.hash": file.hash, "metadata.userId": userId }); + const identicalFileArray = await identicalFile.toArray(); + if(identicalFileArray.length > 0){ + const fileId = identicalFileArray[0]._id; + const fileName = identicalFileArray[0].filename; + res.status(200).json({message: "Reusing file in database!", fileId: fileId, fileName: fileName} as any); + return; + } + const fileStream = fs.createReadStream(file.filepath); // Upload the file to GridFS const uploadStream = bucket.openUploadStream(file.originalFilename || "uploadedFile", { - metadata: { expId: expId }, - }) as Writable; + metadata: { userId: userId, hash: file.hash, lastUsedDate: new Date()}, + }); // Pipe the file stream to GridFS fileStream.pipe(uploadStream).on("finish", () => { - res.status(200).json({ message: "File and ID uploaded successfully." } as any); + res.status(200).json({ message: "File and ID uploaded successfully.", fileId: uploadStream.id, fileName: uploadStream.filename } as any); }); + return; } catch (error) { const message = "Failed to upload experiment file!"; console.error("Error writing experiment file."); + console.log(error); res.status(500).json({ response: message } as any); } } diff --git a/apps/runner/runner.py b/apps/runner/runner.py index 777f9b58..bda4b21c 100644 --- a/apps/runner/runner.py +++ b/apps/runner/runner.py @@ -181,12 +181,11 @@ def download_experiment_files(experiment: ExperimentData): os.makedirs('ResCsvs') explogger.info(f'Downloading file for {experiment.expId}') - filepath = f'experiment{experiment.expId}' - experiment.file = filepath + filepath = experiment.file explogger.info(f"Downloading {filepath} to ExperimentFiles/{experiment.expId}/{filepath}") try: # try to call the backend to download - url = f'http://glados-service-backend:{os.getenv("BACKEND_PORT")}/downloadExpFile?expId={experiment.expId}' + url = f'http://glados-service-backend:{os.getenv("BACKEND_PORT")}/downloadExpFile?fileId={experiment.file}' response = requests.get(url, timeout=60) file_contents = response.content # write the file contents to file path diff --git a/kubernetes_init/backend/deployment-backend.yaml b/kubernetes_init/backend/deployment-backend.yaml index 3114468d..a352331d 100644 --- a/kubernetes_init/backend/deployment-backend.yaml +++ b/kubernetes_init/backend/deployment-backend.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: deployment-test-backend + name: glados-backend namespace: default spec: replicas: 1 diff --git a/kubernetes_init/frontend/deployment-frontend.yaml b/kubernetes_init/frontend/deployment-frontend.yaml index 0df77d22..b2ad1a52 100644 --- a/kubernetes_init/frontend/deployment-frontend.yaml +++ b/kubernetes_init/frontend/deployment-frontend.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: deployment-test-frontend + name: glados-frontend namespace: default spec: replicas: 1