Merge pull request #86 from LuCEresearchlab/push-new-dataset-better-h…

…andling Push new dataset better handling
LuCEresearchlab · Apr 25, 2021 · 9ebdaaa · 9ebdaaa
2 parents be431bb + ecda833
commit 9ebdaaa
Show file tree

Hide file tree

Showing 23 changed files with 302 additions and 201 deletions.
diff --git a/.env b/.env
@@ -1,2 +1,4 @@
 FLASK_APP=flaskr
 FLASK_ENV=development
+NR_WORKERS=2
+NR_THREADS=5
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -22,12 +22,14 @@ services:
     environment:
       - FLASK_APP=${FLASK_APP}
       - FLASK_ENV=${FLASK_ENV}
+      - NR_WORKERS=${NR_WORKERS}
+      - NR_THREADS=${NR_THREADS}
     depends_on:
       - tagging-database
 
 
   tagging-database:
-    image: 'mongo:4.4.3'
+    image: 'mongo:4.2.13-bionic'
     ports:
     - "27017:27017"
     volumes:

diff --git a/frontend/Dockerfile b/frontend/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:15.12-alpine3.10
+FROM node:15.14.0-alpine3.10
 
 EXPOSE 8080
 

diff --git a/frontend/src/components/v2/TaggingUI.tsx b/frontend/src/components/v2/TaggingUI.tsx
@@ -71,7 +71,7 @@ function TaggingUI({taggingSession, dispatchTaggingSession, taggingClusterSessio
     }
 
     const clusterFetch = useFetch<any>(
-        `${TAGGING_SERVICE_URL}/datasets/clusters/dataset/${taggingClusterSession.dataset_id
+        `${TAGGING_SERVICE_URL}/clusters/dataset/${taggingClusterSession.dataset_id
         }/question/${taggingClusterSession.question_id
         }/user/${taggingClusterSession.user_id}`)
 

diff --git a/frontend/src/helpers/PostHelper.ts b/frontend/src/helpers/PostHelper.ts
@@ -45,6 +45,6 @@ export function postClusters(dataset_id: string,
                              question_id: string,
                              user_id: string,
                              clusters: Answer[][]) {
-    post(`${TAGGING_SERVICE_URL}/datasets/clusters/dataset/${dataset_id}/question/${question_id}/user/${user_id}`,
+    post(`${TAGGING_SERVICE_URL}/clusters/dataset/${dataset_id}/question/${question_id}/user/${user_id}`,
         clusters)
 }
diff --git a/frontend/src/index.html b/frontend/src/index.html
@@ -2,7 +2,7 @@
 <html>
   <head>
     <meta charset="utf-8" />
-    <title>My app</title>
+    <title>TSR</title>
   </head>
   <body style="margin:0;">
     <div id="root"></div>

diff --git a/frontend/src/interfaces/Dataset.ts b/frontend/src/interfaces/Dataset.ts
@@ -1,10 +1,18 @@
 export interface Dataset {
-    name: string,
+    "name": string,
     "creation_data": Date,
     "dataset_id": string,
     "questions": [Question]
 }
 
+export interface DatasetDesc {
+    "name": string,
+    "creation_data": Date,
+    "dataset_id": string,
+    "nr_questions": number,
+    "clusters_computed": number
+}
+
 export interface Question {
     "question_id": string,
     "text": string

diff --git a/frontend/src/pages/tagging/DatasetSelection.tsx b/frontend/src/pages/tagging/DatasetSelection.tsx
@@ -1,10 +1,11 @@
-import React, {useState} from 'react';
-import {Table, TableBody, TableContainer, TableHead, TableRow, Paper, Button} from '@material-ui/core';
+import React, {useEffect, useState} from 'react';
+import {Table, TableBody, TableContainer, TableHead, TableRow, Paper, Button, LinearProgress} from '@material-ui/core';
 import {JSONLoader} from '../../helpers/LoaderHelper';
 import {useHistory} from 'react-router-dom'
 import {StyledTableRow, StyledTableCell, useStyles} from "../../components/styled/StyledTable";
 import {downloadDatasetHelper} from "../../helpers/DownloadHelper";
 import {Assignment, AssignmentLate, CloudDownload} from "@material-ui/icons";
+import {DatasetDesc} from "../../interfaces/Dataset";
 
 
 const {TAGGING_SERVICE_URL} = require('../../../config.json')
@@ -30,8 +31,8 @@ function requestUserId() {
 function DatasetSelection() {
     const router = useHistory() // TODO: use router from next once integrated, fix import
 
-    const [datasets, setDatasets] = useState([])
-    const [loaded, setLoaded] = useState(false)
+    const [datasets, setDatasets] = useState<DatasetDesc[]>([])
+    const [loaded, setLoaded] = useState<boolean>(false)
 
     const classes = useStyles();
     const url = TAGGING_SERVICE_URL + "/datasets/list"
@@ -43,6 +44,14 @@ function DatasetSelection() {
         setLoaded(true)
     }
 
+    useEffect(() => {
+        const interval = setInterval(() => setLoaded(false), 5000);
+        return () => {
+            clearInterval(interval);
+        };
+    }, []);
+
+
     return (
         <TableContainer component={Paper} style={{
             width: '98%',
@@ -57,45 +66,82 @@ function DatasetSelection() {
                     </TableRow>
                 </TableHead>
                 <TableBody>
-                    {datasets.map((row: { id: string, name: string, date: string }) => (
-                        <StyledTableRow key={row.id}>
-                            <StyledTableCell component="th" scope="row" onClick={
-                                () => redirect(row.id, "/taggingUI/tagView/", router)
-                            }>
-                                {row.name}
-                            </StyledTableCell>
-                            <StyledTableCell align="right" onClick={
-                                () => redirect(row.id, "/taggingUI/tagView/", router)
-                            }>{row.date}</StyledTableCell>
-                            <StyledTableCell align={"right"}>
-                                <Button
-                                    title={"Download"}
-                                    variant="outlined" color="primary"
-                                    href="#outlined-buttons"
-                                    onClick={() => downloadDatasetHelper(row.id, row.name)}>
-                                    <CloudDownload/>
-                                </Button>
-                                <Button
-                                    title={"Summary"}
-                                    variant="outlined"
-                                    color="primary"
-                                    onClick={
-                                        () => redirect(row.id, "/taggingUI/summary/", router)
-                                    }>
-                                    <Assignment/>
-                                </Button>
-                                <Button
-                                    title={"Diff Data"}
-                                    variant="outlined"
-                                    color="primary"
-                                    onClick={
-                                        () => redirect(row.id, "/taggingUI/mergeView/", router)
-                                    }>
-                                    <AssignmentLate/>
-                                </Button>
-                            </StyledTableCell>
-                        </StyledTableRow>
-                    ))}
+                    {datasets.map((dataset: DatasetDesc) => {
+                        const loading_cluster = dataset.clusters_computed != dataset.nr_questions
+                        // const needed_time_s = 1000 * 60 * 2 * dataset.nr_questions
+                        // const started = new Date(dataset.creation_data)
+                        // const now = new Date()
+                        //
+                        // const time_left = needed_time_s - (now.getTime() - started.getTime()) - now.getTimezoneOffset() * 1000 * 60
+                        //
+                        // const time_left_minutes = Math.floor(time_left / (60 * 1000))
+                        // const seconds = Math.floor((time_left - time_left_minutes * (60 * 1000)) / (1000))
+                        //
+                        // const time_left_seconds = seconds < 10 ? '0' + seconds : seconds
+
+                        if (dataset.clusters_computed != dataset.nr_questions) {
+                            return (
+                                <StyledTableRow key={dataset.dataset_id}>
+                                    <StyledTableCell component={'th'} scope={'row'}>
+                                        {dataset.name}
+                                    </StyledTableCell>
+                                    <StyledTableCell component={'th'} scope={'row'}>
+                                        <LinearProgress
+                                            variant={'determinate'}
+                                            value={Math.ceil(100 * dataset.clusters_computed / dataset.nr_questions)}
+                                        />
+                                    </StyledTableCell>
+                                    <StyledTableCell component={'th'} scope={'row'} style={{textAlign: 'end'}}>
+                                        {`${dataset.clusters_computed}/${dataset.nr_questions}`}
+                                    </StyledTableCell>
+                                </StyledTableRow>
+                            )
+                        }
+
+                        return (
+                            <StyledTableRow key={dataset.dataset_id}>
+                                <StyledTableCell component="th" scope="row" onClick={
+                                    () => redirect(dataset.dataset_id, "/taggingUI/tagView/", router)}>
+                                    {dataset.name}
+                                </StyledTableCell>
+                                <StyledTableCell align="right" onClick={
+                                    () => redirect(dataset.dataset_id, "/taggingUI/tagView/", router)
+                                }>
+                                    {dataset.creation_data}
+                                </StyledTableCell>
+                                <StyledTableCell align={"right"}>
+                                    <Button
+                                        title={"Download"}
+                                        variant="outlined" color="primary"
+                                        href="#outlined-buttons"
+                                        disabled={loading_cluster}
+                                        onClick={() => downloadDatasetHelper(dataset.dataset_id, dataset.name)}>
+                                        <CloudDownload/>
+                                    </Button>
+                                    <Button
+                                        title={"Summary"}
+                                        variant="outlined"
+                                        disabled={loading_cluster}
+                                        color="primary"
+                                        onClick={
+                                            () => redirect(dataset.dataset_id, "/taggingUI/summary/", router)
+                                        }>
+                                        <Assignment/>
+                                    </Button>
+                                    <Button
+                                        title={"Diff Data"}
+                                        variant="outlined"
+                                        disabled={loading_cluster}
+                                        color="primary"
+                                        onClick={
+                                            () => redirect(dataset.dataset_id, "/taggingUI/mergeView/", router)
+                                        }>
+                                        <AssignmentLate/>
+                                    </Button>
+                                </StyledTableCell>
+                            </StyledTableRow>
+                        )
+                    })}
                 </TableBody>
             </Table>
         </TableContainer>

diff --git a/tagging-service/.dockerignore b/tagging-service/.dockerignore
@@ -1,2 +1,3 @@
 test
+cache
 *.iml
diff --git a/tagging-service/Dockerfile b/tagging-service/Dockerfile
@@ -8,4 +8,4 @@ COPY . /tagging_service
 
 RUN pip install -r requirements.txt
 
-CMD flask run --host=0.0.0.0
+CMD gunicorn --preload --workers $NR_WORKERS --reload --timeout 90 -b 0.0.0.0:5000 flaskr:my_app
diff --git a/tagging-service/flaskr/__init__.py b/tagging-service/flaskr/__init__.py
@@ -28,3 +28,5 @@ def handle_invalid_usage(error):
 if __name__ == "__main__":
     app = create_app()
     app.run(debug=True, host='0.0.0.0', threaded=True)
+
+my_app = create_app()
diff --git a/tagging-service/flaskr/config/__init__.py b/tagging-service/flaskr/config/__init__.py
@@ -4,8 +4,9 @@
 # https://flask-caching.readthedocs.io/en/latest/
 config = {
     "DEBUG": True,  # some Flask specific configs
-    "CACHE_TYPE": "simple",  # Flask-Caching related configs
+    "CACHE_TYPE": "filesystem",  # Flask-Caching related configs
     "CACHE_DIR": "cache",
+    "CACHE_THRESHOLD": 1000,
     "CACHE_DEFAULT_TIMEOUT": 0,
 }
 cache = Cache(config=config)
diff --git a/tagging-service/flaskr/endpoints/__init__.py b/tagging-service/flaskr/endpoints/__init__.py
@@ -2,6 +2,7 @@
 from flaskr.endpoints.upload_api import api as ns1
 from flaskr.endpoints.progmiscon_api import api as ns2
 from flaskr.endpoints.datasets_api import api as ns3
+from flaskr.endpoints.cluster_api import api as ns4
 
 api = Api(version='1.0',
           title='tagging Service',
@@ -12,3 +13,4 @@
 api.add_namespace(ns1)
 api.add_namespace(ns2)
 api.add_namespace(ns3)
+api.add_namespace(ns4)
diff --git a/tagging-service/flaskr/endpoints/cluster_api.py b/tagging-service/flaskr/endpoints/cluster_api.py
@@ -0,0 +1,36 @@
+from flask import request
+from flask_restx import Namespace, Resource, fields
+from flaskr.endpoints.upload_api import ANSWER
+from flaskr import cache
+from flaskr.util.mongo_helper import get_cluster, save_cluster
+
+api = Namespace('clusters', description='API to view available datasets')
+
+
+CLUSTERED_ANSWERS = api.model('Clustered Answers', {
+    'dataset_id': fields.String(required=True, readonly=True, description='ID of the dataset',
+                                example='603501f39175ac3898e094cc'),
+    'question_id': fields.String(required=True, readonly=True, description='ID of the question',
+                                 example='6035089963cf6ef09a9c418e'),
+    'clusters': fields.List(fields.List(fields.Nested(ANSWER)))
+})
+
+
+@api.route('/dataset/<string:dataset_id>/question/<string:question_id>/user/<string:user_id>')
+@api.doc(description='API to get clusters for current user',
+         params={
+             'dataset_id': 'ID of the dataset',
+             'user_id': 'ID of the user'
+         })
+class Clusters(Resource):
+    @api.marshal_list_with(CLUSTERED_ANSWERS)
+    @cache.memoize()
+    def get(self, dataset_id, question_id, user_id):
+        return get_cluster(dataset_id=dataset_id, question_id=question_id, user_id=user_id)[0]
+
+    @api.expect(CLUSTERED_ANSWERS)
+    def post(self, dataset_id, question_id, user_id):
+        data = request.get_json()
+        save_cluster(dataset_id=dataset_id, question_id=question_id, user_id=user_id,
+                     cluster=data)
+        cache.delete_memoized(Clusters.get, dataset_id, question_id, user_id)
diff --git a/tagging-service/flaskr/endpoints/datasets_api.py b/tagging-service/flaskr/endpoints/datasets_api.py
@@ -2,8 +2,6 @@
 from flask_restx import Namespace, Resource, fields
 
 import flaskr.util.mongo_helper as db
-from flaskr.endpoints.upload_api import ANSWER
-from flaskr import cache
 
 api = Namespace('datasets', description='API to view available datasets')
 
@@ -40,13 +38,6 @@
                        example='603501f39175ac3898e094cc')
 })
 
-CLUSTERED_ANSWERS = api.model('Clustered Answers', {
-    'dataset_id': fields.String(required=True, readonly=True, description='ID of the dataset',
-                                example='603501f39175ac3898e094cc'),
-    'question_id': fields.String(required=True, readonly=True, description='ID of the question',
-                                 example='6035089963cf6ef09a9c418e'),
-    'clusters': fields.List(fields.List(fields.Nested(ANSWER)))
-})
 
 
 @api.route('/tagged-datasets')
@@ -124,20 +115,3 @@ def get(self, dataset_id, question_id):
         return db.get_tagged_question(dataset_id=dataset_id, question_id=question_id)
 
 
-@api.route('/clusters/dataset/<string:dataset_id>/question/<string:question_id>/user/<string:user_id>')
-@api.doc(description='API to get clusters for current user',
-         params={
-             'dataset_id': 'ID of the dataset',
-             'user_id': 'ID of the user'
-         })
-class Clusters(Resource):
-    @api.marshal_list_with(CLUSTERED_ANSWERS)
-    @cache.memoize()
-    def get(self, dataset_id, question_id, user_id):
-        return db.get_cluster(dataset_id=dataset_id, question_id=question_id, user_id=user_id)[0]
-
-    @api.expect(CLUSTERED_ANSWERS)
-    def post(self, dataset_id, question_id, user_id):
-        data = request.get_json()
-        db.save_cluster(dataset_id=dataset_id, question_id=question_id, user_id=user_id, cluster=data)
-        cache.delete_memoized(Clusters.get, dataset_id, user_id)
-Original file line number
+Diff line change
@@ -1,2 +1,3 @@
     test
+    cache
     *.iml
Original file line number	Diff line number	Diff line change
Expand Up		@@ -8,4 +8,4 @@ COPY . /tagging_service

		RUN pip install -r requirements.txt

		CMD flask run --host=0.0.0.0
		CMD gunicorn --preload --workers $NR_WORKERS --reload --timeout 90 -b 0.0.0.0:5000 flaskr:my_app